mod-wsgi-telemetry 1.0.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mod_wsgi/__init__.py +2 -0
- mod_wsgi/telemetry/__init__.py +1 -0
- mod_wsgi/telemetry/cli.py +55 -0
- mod_wsgi/telemetry/contention.py +229 -0
- mod_wsgi/telemetry/dump.py +102 -0
- mod_wsgi/telemetry/ingest.py +773 -0
- mod_wsgi/telemetry/server.py +270 -0
- mod_wsgi/telemetry/simulate.py +649 -0
- mod_wsgi/telemetry/static/index.html +8368 -0
- mod_wsgi/telemetry/tui.py +1536 -0
- mod_wsgi/telemetry/wire.py +480 -0
- mod_wsgi_telemetry-1.0.0.dev2.dist-info/METADATA +79 -0
- mod_wsgi_telemetry-1.0.0.dev2.dist-info/RECORD +16 -0
- mod_wsgi_telemetry-1.0.0.dev2.dist-info/WHEEL +4 -0
- mod_wsgi_telemetry-1.0.0.dev2.dist-info/entry_points.txt +2 -0
- mod_wsgi_telemetry-1.0.0.dev2.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,1536 @@
|
|
|
1
|
+
"""Terminal monitor for mod_wsgi telemetry.
|
|
2
|
+
|
|
3
|
+
Connects to a running ``mod_wsgi-telemetry serve`` ingester's WebSocket and
|
|
4
|
+
renders the live state as a top-style terminal UI. Multiple views switchable
|
|
5
|
+
by keystroke; the persistent header stays visible across views.
|
|
6
|
+
|
|
7
|
+
Usage:
|
|
8
|
+
mod_wsgi-telemetry top # connect to localhost
|
|
9
|
+
mod_wsgi-telemetry top --url ws://host:8888/ws
|
|
10
|
+
mod_wsgi-telemetry top --view slow # start on slow-requests view
|
|
11
|
+
mod_wsgi-telemetry top --once # render one frame and exit
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import argparse
|
|
17
|
+
import asyncio
|
|
18
|
+
import curses
|
|
19
|
+
import json
|
|
20
|
+
import logging
|
|
21
|
+
import math
|
|
22
|
+
import os
|
|
23
|
+
import signal
|
|
24
|
+
import sys
|
|
25
|
+
import time
|
|
26
|
+
from collections import deque
|
|
27
|
+
from dataclasses import dataclass, field
|
|
28
|
+
from typing import Any
|
|
29
|
+
|
|
30
|
+
import aiohttp
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
PHASES = ["server", "queue", "daemon", "application", "request"]
|
|
34
|
+
|
|
35
|
+
# HDR histogram layout. Mirrors the daemon-side encoder: 16 octaves, each
|
|
36
|
+
# split into 4 sub-buckets, plus 1 overflow bucket = 65 entries. Bucket 0
|
|
37
|
+
# covers [1ms, 1.25ms), bucket 1 [1.25ms, 1.5ms), and so on; bucket 64 is
|
|
38
|
+
# >65.5s.
|
|
39
|
+
HDR_OCTAVES = 16
|
|
40
|
+
HDR_SUBS = 4
|
|
41
|
+
HDR_TOTAL = HDR_OCTAVES * HDR_SUBS + 1
|
|
42
|
+
HDR_BASE_SECONDS = 0.001 # bucket 0 lower bound: 1 ms expressed as seconds
|
|
43
|
+
|
|
44
|
+
DEFAULT_URL = "ws://127.0.0.1:8888/ws"
|
|
45
|
+
SAMPLE_RETENTION = 600 # max samples per pid we keep client-side
|
|
46
|
+
REFRESH_RATES = [0.5, 1.0, 2.0, 5.0]
|
|
47
|
+
WINDOW_CHOICES = [10, 60, 600] # rolling window in seconds for overview/latency
|
|
48
|
+
|
|
49
|
+
VIEWS = ["overview", "processes", "workers", "latency", "slow"]
|
|
50
|
+
VIEW_KEYS = {
|
|
51
|
+
ord("o"): "overview", ord("1"): "overview",
|
|
52
|
+
ord("p"): "processes", ord("2"): "processes",
|
|
53
|
+
ord("w"): "workers", ord("3"): "workers",
|
|
54
|
+
ord("l"): "latency", ord("4"): "latency",
|
|
55
|
+
ord("s"): "slow", ord("5"): "slow",
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
SPARK_BLOCKS = " ▁▂▃▄▅▆▇█"
|
|
59
|
+
BAR_FULL = "█"
|
|
60
|
+
BAR_EMPTY = "░"
|
|
61
|
+
|
|
62
|
+
# Color pair IDs.
|
|
63
|
+
CP_HEADER = 1
|
|
64
|
+
CP_OK = 2
|
|
65
|
+
CP_WARN = 3
|
|
66
|
+
CP_CRIT = 4
|
|
67
|
+
CP_DIM = 5
|
|
68
|
+
CP_BAR = 6
|
|
69
|
+
CP_TAB = 7
|
|
70
|
+
CP_TAB_ACTIVE = 8
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# ---------------------------------------------------------------------------
|
|
74
|
+
# State mirror
|
|
75
|
+
# ---------------------------------------------------------------------------
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass
|
|
79
|
+
class ProcState:
|
|
80
|
+
"""Client-side mirror of the ingester's per-pid rolling window."""
|
|
81
|
+
pid: int
|
|
82
|
+
hostname: str = ""
|
|
83
|
+
process_group: str = ""
|
|
84
|
+
mod_wsgi_version: str = ""
|
|
85
|
+
python_version: str = ""
|
|
86
|
+
apache_version: str = ""
|
|
87
|
+
mpm_name: str = ""
|
|
88
|
+
sample_period: float = 1.0
|
|
89
|
+
samples: deque = field(default_factory=lambda: deque(maxlen=SAMPLE_RETENTION))
|
|
90
|
+
last_seen: float = 0.0
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@dataclass
|
|
94
|
+
class SlowEntry:
|
|
95
|
+
pid: int
|
|
96
|
+
thread_id: int
|
|
97
|
+
log_id: str
|
|
98
|
+
method: str
|
|
99
|
+
scheme: str
|
|
100
|
+
hostname: str
|
|
101
|
+
script_name: str
|
|
102
|
+
path_info: str
|
|
103
|
+
start_stamp: float
|
|
104
|
+
duration: float
|
|
105
|
+
state: int
|
|
106
|
+
input_bytes: int
|
|
107
|
+
input_reads: int
|
|
108
|
+
output_bytes: int
|
|
109
|
+
output_writes: int
|
|
110
|
+
cpu_user_time: float
|
|
111
|
+
cpu_system_time: float
|
|
112
|
+
status: int = 0
|
|
113
|
+
last_seen: float = 0.0
|
|
114
|
+
|
|
115
|
+
@classmethod
|
|
116
|
+
def from_dict(cls, d: dict) -> "SlowEntry":
|
|
117
|
+
return cls(
|
|
118
|
+
pid=int(d["pid"]),
|
|
119
|
+
thread_id=int(d["thread_id"]),
|
|
120
|
+
log_id=d["log_id"],
|
|
121
|
+
method=d["method"],
|
|
122
|
+
scheme=d["scheme"],
|
|
123
|
+
hostname=d["hostname"],
|
|
124
|
+
script_name=d["script_name"],
|
|
125
|
+
path_info=d["path_info"],
|
|
126
|
+
start_stamp=float(d["start_stamp"]),
|
|
127
|
+
duration=float(d["duration"]),
|
|
128
|
+
state=int(d["state"]),
|
|
129
|
+
input_bytes=int(d.get("input_bytes", 0)),
|
|
130
|
+
input_reads=int(d.get("input_reads", 0)),
|
|
131
|
+
output_bytes=int(d.get("output_bytes", 0)),
|
|
132
|
+
output_writes=int(d.get("output_writes", 0)),
|
|
133
|
+
cpu_user_time=float(d.get("cpu_user_time", 0.0)),
|
|
134
|
+
cpu_system_time=float(d.get("cpu_system_time", 0.0)),
|
|
135
|
+
status=int(d.get("status", 0)),
|
|
136
|
+
last_seen=time.monotonic(),
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
def url(self) -> str:
|
|
140
|
+
path = (self.script_name or "") + (self.path_info or "")
|
|
141
|
+
return path or "/"
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class State:
|
|
145
|
+
"""Snapshot + live-sample mirror of the ingester."""
|
|
146
|
+
|
|
147
|
+
def __init__(self) -> None:
|
|
148
|
+
self.processes: dict[int, ProcState] = {}
|
|
149
|
+
self.slow: dict[tuple, SlowEntry] = {}
|
|
150
|
+
self.connected: bool = False
|
|
151
|
+
self.url: str = ""
|
|
152
|
+
self.last_message: float = 0.0
|
|
153
|
+
self.last_error: str = ""
|
|
154
|
+
self.total_received: int = 0
|
|
155
|
+
self.decode_errors: int = 0
|
|
156
|
+
|
|
157
|
+
def apply(self, msg: dict) -> None:
|
|
158
|
+
t = msg.get("type")
|
|
159
|
+
if t == "snapshot":
|
|
160
|
+
self._apply_snapshot(msg)
|
|
161
|
+
elif t == "sample":
|
|
162
|
+
self._apply_sample(msg)
|
|
163
|
+
elif t == "slow_request":
|
|
164
|
+
self._apply_slow(msg)
|
|
165
|
+
elif t == "slow_clear":
|
|
166
|
+
self._apply_slow_clear(msg)
|
|
167
|
+
self.last_message = time.monotonic()
|
|
168
|
+
|
|
169
|
+
def _apply_snapshot(self, msg: dict) -> None:
|
|
170
|
+
self.total_received = int(msg.get("total_received", 0))
|
|
171
|
+
self.decode_errors = int(msg.get("decode_errors", 0))
|
|
172
|
+
self.processes.clear()
|
|
173
|
+
for p in msg.get("processes", []):
|
|
174
|
+
pid = int(p["pid"])
|
|
175
|
+
ps = ProcState(
|
|
176
|
+
pid=pid,
|
|
177
|
+
hostname=p.get("hostname", ""),
|
|
178
|
+
process_group=p.get("process_group", ""),
|
|
179
|
+
mod_wsgi_version=p.get("mod_wsgi_version", ""),
|
|
180
|
+
python_version=p.get("python_version", ""),
|
|
181
|
+
apache_version=p.get("apache_version", ""),
|
|
182
|
+
mpm_name=p.get("mpm_name", ""),
|
|
183
|
+
last_seen=time.monotonic(),
|
|
184
|
+
)
|
|
185
|
+
for s in p.get("samples", []):
|
|
186
|
+
ps.samples.append(s)
|
|
187
|
+
fields = s.get("fields", {})
|
|
188
|
+
sp = fields.get("sample_period")
|
|
189
|
+
if isinstance(sp, (int, float)) and sp > 0:
|
|
190
|
+
ps.sample_period = float(sp)
|
|
191
|
+
self.processes[pid] = ps
|
|
192
|
+
self.slow.clear()
|
|
193
|
+
for r in msg.get("slow_requests", []):
|
|
194
|
+
entry = SlowEntry.from_dict(r["entry"])
|
|
195
|
+
key = tuple(r["key"])
|
|
196
|
+
self.slow[key] = entry
|
|
197
|
+
|
|
198
|
+
def _apply_sample(self, msg: dict) -> None:
|
|
199
|
+
self.total_received += 1
|
|
200
|
+
pid = int(msg["pid"])
|
|
201
|
+
ps = self.processes.get(pid)
|
|
202
|
+
if ps is None:
|
|
203
|
+
ps = ProcState(pid=pid)
|
|
204
|
+
self.processes[pid] = ps
|
|
205
|
+
ps.samples.append(msg)
|
|
206
|
+
ps.last_seen = time.monotonic()
|
|
207
|
+
fields = msg.get("fields", {})
|
|
208
|
+
|
|
209
|
+
def _latch(name: str, attr: str) -> None:
|
|
210
|
+
v = fields.get(name)
|
|
211
|
+
if isinstance(v, str) and v:
|
|
212
|
+
setattr(ps, attr, v)
|
|
213
|
+
|
|
214
|
+
_latch("hostname", "hostname")
|
|
215
|
+
_latch("process_group", "process_group")
|
|
216
|
+
_latch("mod_wsgi_version", "mod_wsgi_version")
|
|
217
|
+
_latch("python_version", "python_version")
|
|
218
|
+
_latch("apache_version", "apache_version")
|
|
219
|
+
_latch("mpm_name", "mpm_name")
|
|
220
|
+
|
|
221
|
+
sp = fields.get("sample_period")
|
|
222
|
+
if isinstance(sp, (int, float)) and sp > 0:
|
|
223
|
+
ps.sample_period = float(sp)
|
|
224
|
+
|
|
225
|
+
def _apply_slow(self, msg: dict) -> None:
|
|
226
|
+
key = tuple(msg["key"])
|
|
227
|
+
self.slow[key] = SlowEntry.from_dict(msg["entry"])
|
|
228
|
+
|
|
229
|
+
def _apply_slow_clear(self, msg: dict) -> None:
|
|
230
|
+
kept = {tuple(k["key"]): SlowEntry.from_dict(k["entry"])
|
|
231
|
+
for k in msg.get("kept", [])}
|
|
232
|
+
self.slow = kept
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
# ---------------------------------------------------------------------------
|
|
236
|
+
# Aggregation helpers
|
|
237
|
+
# ---------------------------------------------------------------------------
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _samples_in_window(ps: ProcState, seconds: float) -> list[dict]:
|
|
241
|
+
"""Newest-first samples within the given wall-clock window."""
|
|
242
|
+
if not ps.samples:
|
|
243
|
+
return []
|
|
244
|
+
newest = ps.samples[-1].get("stamp", 0.0)
|
|
245
|
+
cutoff = newest - seconds
|
|
246
|
+
out = []
|
|
247
|
+
for s in reversed(ps.samples):
|
|
248
|
+
if s.get("stamp", 0.0) < cutoff:
|
|
249
|
+
break
|
|
250
|
+
out.append(s)
|
|
251
|
+
return out
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _last_field(ps: ProcState, name: str, default=0):
|
|
255
|
+
if not ps.samples:
|
|
256
|
+
return default
|
|
257
|
+
return ps.samples[-1].get("fields", {}).get(name, default)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _hdr_bucket_bounds(idx: int) -> tuple[float, float]:
|
|
261
|
+
"""Lower/upper bound (seconds) of bucket `idx`."""
|
|
262
|
+
if idx >= HDR_OCTAVES * HDR_SUBS:
|
|
263
|
+
return (HDR_BASE_SECONDS * (1 << HDR_OCTAVES), float("inf"))
|
|
264
|
+
octave = idx // HDR_SUBS
|
|
265
|
+
sub = idx % HDR_SUBS
|
|
266
|
+
octave_lo = HDR_BASE_SECONDS * (1 << octave)
|
|
267
|
+
sub_width = octave_lo / HDR_SUBS
|
|
268
|
+
return (octave_lo + sub * sub_width, octave_lo + (sub + 1) * sub_width)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _hdr_percentile(buckets: list[int], pct: float) -> float | None:
|
|
272
|
+
"""Linear interpolation inside the bucket containing the percentile."""
|
|
273
|
+
total = sum(buckets)
|
|
274
|
+
if total <= 0:
|
|
275
|
+
return None
|
|
276
|
+
target = pct * total / 100.0
|
|
277
|
+
cumulative = 0
|
|
278
|
+
for i, c in enumerate(buckets):
|
|
279
|
+
if c <= 0:
|
|
280
|
+
continue
|
|
281
|
+
if cumulative + c >= target:
|
|
282
|
+
lo, hi = _hdr_bucket_bounds(i)
|
|
283
|
+
if math.isinf(hi):
|
|
284
|
+
return lo
|
|
285
|
+
frac = (target - cumulative) / c
|
|
286
|
+
return lo + frac * (hi - lo)
|
|
287
|
+
cumulative += c
|
|
288
|
+
return None
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _aggregate_buckets(
|
|
292
|
+
state: State, phase: str, seconds: float, group_filter: str | None
|
|
293
|
+
) -> list[int]:
|
|
294
|
+
"""Sum HDR buckets for `phase` across processes within the window."""
|
|
295
|
+
name = f"{phase}_time_buckets"
|
|
296
|
+
out = [0] * HDR_TOTAL
|
|
297
|
+
for ps in state.processes.values():
|
|
298
|
+
if group_filter and ps.process_group != group_filter:
|
|
299
|
+
continue
|
|
300
|
+
for s in _samples_in_window(ps, seconds):
|
|
301
|
+
buckets = s.get("fields", {}).get(name)
|
|
302
|
+
if isinstance(buckets, list) and len(buckets) == HDR_TOTAL:
|
|
303
|
+
for i, c in enumerate(buckets):
|
|
304
|
+
out[i] += c
|
|
305
|
+
return out
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _phase_min_max(
|
|
309
|
+
state: State, phase: str, seconds: float, group_filter: str | None
|
|
310
|
+
) -> tuple[float | None, float | None]:
|
|
311
|
+
"""Min-of-mins, max-of-maxes (seconds) for `phase` in window."""
|
|
312
|
+
min_name = f"{phase}_time_min"
|
|
313
|
+
max_name = f"{phase}_time_max"
|
|
314
|
+
lo: float | None = None
|
|
315
|
+
hi: float | None = None
|
|
316
|
+
for ps in state.processes.values():
|
|
317
|
+
if group_filter and ps.process_group != group_filter:
|
|
318
|
+
continue
|
|
319
|
+
for s in _samples_in_window(ps, seconds):
|
|
320
|
+
f = s.get("fields", {})
|
|
321
|
+
v = f.get(min_name)
|
|
322
|
+
if isinstance(v, (int, float)) and v > 0:
|
|
323
|
+
lo = v if lo is None else min(lo, v)
|
|
324
|
+
v = f.get(max_name)
|
|
325
|
+
if isinstance(v, (int, float)) and v > 0:
|
|
326
|
+
hi = v if hi is None else max(hi, v)
|
|
327
|
+
return lo, hi
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _process_groups(state: State) -> list[str]:
|
|
331
|
+
seen = sorted({ps.process_group for ps in state.processes.values()
|
|
332
|
+
if ps.process_group})
|
|
333
|
+
return seen
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
# ---------------------------------------------------------------------------
|
|
337
|
+
# Formatters
|
|
338
|
+
# ---------------------------------------------------------------------------
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def fmt_bytes(n: float) -> str:
|
|
342
|
+
if n < 1024:
|
|
343
|
+
return f"{int(n)}B"
|
|
344
|
+
for unit in ("KB", "MB", "GB", "TB"):
|
|
345
|
+
n /= 1024
|
|
346
|
+
if n < 1024:
|
|
347
|
+
return f"{n:.1f}{unit}"
|
|
348
|
+
return f"{n:.1f}PB"
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def fmt_seconds(s: float | int | None) -> str:
|
|
352
|
+
"""Render a duration in seconds as a human-readable string."""
|
|
353
|
+
if s is None:
|
|
354
|
+
return "-"
|
|
355
|
+
if s < 0.001:
|
|
356
|
+
return f"{s*1_000_000:.0f}µs"
|
|
357
|
+
if s < 1.0:
|
|
358
|
+
return f"{s*1000:.1f}ms"
|
|
359
|
+
return f"{s:.2f}s"
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def sparkline(values: list[float], width: int) -> str:
|
|
363
|
+
if width <= 0 or not values:
|
|
364
|
+
return ""
|
|
365
|
+
if len(values) > width:
|
|
366
|
+
# downsample by averaging windows
|
|
367
|
+
step = len(values) / width
|
|
368
|
+
out = []
|
|
369
|
+
for i in range(width):
|
|
370
|
+
lo = int(i * step)
|
|
371
|
+
hi = max(lo + 1, int((i + 1) * step))
|
|
372
|
+
out.append(sum(values[lo:hi]) / max(1, hi - lo))
|
|
373
|
+
values = out
|
|
374
|
+
elif len(values) < width:
|
|
375
|
+
values = [0.0] * (width - len(values)) + values
|
|
376
|
+
vmax = max(values) if values else 0.0
|
|
377
|
+
if vmax <= 0:
|
|
378
|
+
return SPARK_BLOCKS[0] * width
|
|
379
|
+
levels = len(SPARK_BLOCKS) - 1
|
|
380
|
+
return "".join(
|
|
381
|
+
SPARK_BLOCKS[min(levels, max(0, int(v / vmax * levels)))]
|
|
382
|
+
for v in values
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
# ---------------------------------------------------------------------------
|
|
387
|
+
# Curses-safe drawing
|
|
388
|
+
# ---------------------------------------------------------------------------
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def safe_addstr(win, y: int, x: int, text: str, attr: int = 0) -> None:
|
|
392
|
+
"""addstr that won't raise if it bumps the bottom-right cell."""
|
|
393
|
+
try:
|
|
394
|
+
h, w = win.getmaxyx()
|
|
395
|
+
if y < 0 or y >= h or x >= w:
|
|
396
|
+
return
|
|
397
|
+
if x < 0:
|
|
398
|
+
text = text[-x:]
|
|
399
|
+
x = 0
|
|
400
|
+
avail = w - x
|
|
401
|
+
if avail <= 0:
|
|
402
|
+
return
|
|
403
|
+
if len(text) > avail:
|
|
404
|
+
text = text[:avail]
|
|
405
|
+
win.addstr(y, x, text, attr)
|
|
406
|
+
except curses.error:
|
|
407
|
+
pass
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
# ---------------------------------------------------------------------------
|
|
411
|
+
# View renderers
|
|
412
|
+
# ---------------------------------------------------------------------------
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
@dataclass
|
|
416
|
+
class UIState:
|
|
417
|
+
view: str = "overview"
|
|
418
|
+
paused: bool = False
|
|
419
|
+
refresh_idx: int = 1 # 1.0 s
|
|
420
|
+
window_idx: int = 1 # 60 s
|
|
421
|
+
phase_idx: int = 4 # request
|
|
422
|
+
process_sort: str = "rps" # rps|cpu|rss|p95|slow|pid
|
|
423
|
+
slow_sort: str = "elapsed" # elapsed|pid|method|url
|
|
424
|
+
slow_state_filter: int = -1 # -1=any, 0=active, 1=completed
|
|
425
|
+
slow_search: str = ""
|
|
426
|
+
group_filter: str | None = None
|
|
427
|
+
show_help: bool = False
|
|
428
|
+
|
|
429
|
+
@property
|
|
430
|
+
def refresh(self) -> float:
|
|
431
|
+
return REFRESH_RATES[self.refresh_idx]
|
|
432
|
+
|
|
433
|
+
@property
|
|
434
|
+
def window(self) -> int:
|
|
435
|
+
return WINDOW_CHOICES[self.window_idx]
|
|
436
|
+
|
|
437
|
+
@property
|
|
438
|
+
def phase(self) -> str:
|
|
439
|
+
return PHASES[self.phase_idx]
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _aggregate_header(state: State, ui: UIState) -> dict:
|
|
443
|
+
"""Compute the always-visible header values once per render."""
|
|
444
|
+
rps_now = 0.0
|
|
445
|
+
in_bytes_per_s = 0.0
|
|
446
|
+
out_bytes_per_s = 0.0
|
|
447
|
+
cpu_user = 0.0
|
|
448
|
+
cpu_sys = 0.0
|
|
449
|
+
rss_total = 0
|
|
450
|
+
rss_max = 0
|
|
451
|
+
busy = 0
|
|
452
|
+
threads_total = 0
|
|
453
|
+
queue_mean = 0.0
|
|
454
|
+
queue_count = 0
|
|
455
|
+
|
|
456
|
+
rps_window: dict[int, list[tuple[int, int]]] = {}
|
|
457
|
+
status_1xx = 0
|
|
458
|
+
status_2xx = 0
|
|
459
|
+
status_3xx = 0
|
|
460
|
+
status_4xx = 0
|
|
461
|
+
status_5xx = 0
|
|
462
|
+
interval_requests = 0
|
|
463
|
+
for ps in state.processes.values():
|
|
464
|
+
if ui.group_filter and ps.process_group != ui.group_filter:
|
|
465
|
+
continue
|
|
466
|
+
if not ps.samples:
|
|
467
|
+
continue
|
|
468
|
+
last = ps.samples[-1]
|
|
469
|
+
f = last.get("fields", {})
|
|
470
|
+
rps_now += float(f.get("request_throughput") or 0)
|
|
471
|
+
in_bytes_per_s += float(f.get("input_bytes_total") or 0) / max(1e-3, ps.sample_period)
|
|
472
|
+
out_bytes_per_s += float(f.get("output_bytes_total") or 0) / max(1e-3, ps.sample_period)
|
|
473
|
+
cpu_user += float(f.get("cpu_user_utilization") or 0)
|
|
474
|
+
cpu_sys += float(f.get("cpu_system_utilization") or 0)
|
|
475
|
+
rss = int(f.get("memory_rss") or 0)
|
|
476
|
+
rss_total += rss
|
|
477
|
+
rss_max = max(rss_max, int(f.get("memory_max_rss") or 0))
|
|
478
|
+
active = int(f.get("request_threads_active") or 0)
|
|
479
|
+
maximum = int(f.get("request_threads_maximum") or 0)
|
|
480
|
+
busy += active
|
|
481
|
+
threads_total += maximum
|
|
482
|
+
qt = f.get("queue_time")
|
|
483
|
+
if isinstance(qt, (int, float)) and qt > 0:
|
|
484
|
+
queue_mean += qt
|
|
485
|
+
queue_count += 1
|
|
486
|
+
status_1xx += int(f.get("status_1xx_total") or 0)
|
|
487
|
+
status_2xx += int(f.get("status_2xx_total") or 0)
|
|
488
|
+
status_3xx += int(f.get("status_3xx_total") or 0)
|
|
489
|
+
status_4xx += int(f.get("status_4xx_total") or 0)
|
|
490
|
+
status_5xx += int(f.get("status_5xx_total") or 0)
|
|
491
|
+
interval_requests += int(f.get("request_count") or 0)
|
|
492
|
+
|
|
493
|
+
# 1-min and 10-min RPS averages: total request_count over window / seconds.
|
|
494
|
+
rps_1m = _avg_rps(state, ui, 60)
|
|
495
|
+
rps_10m = _avg_rps(state, ui, 600)
|
|
496
|
+
|
|
497
|
+
buckets = _aggregate_buckets(state, "request", ui.window, ui.group_filter)
|
|
498
|
+
p50 = _hdr_percentile(buckets, 50)
|
|
499
|
+
p95 = _hdr_percentile(buckets, 95)
|
|
500
|
+
p99 = _hdr_percentile(buckets, 99)
|
|
501
|
+
lo, hi = _phase_min_max(state, "request", ui.window, ui.group_filter)
|
|
502
|
+
|
|
503
|
+
now = time.time()
|
|
504
|
+
active_slow = sum(1 for e in state.slow.values() if e.state == 0)
|
|
505
|
+
recent_slow = sum(
|
|
506
|
+
1 for e in state.slow.values()
|
|
507
|
+
if (now - e.start_stamp) <= 60
|
|
508
|
+
)
|
|
509
|
+
total_slow = len(state.slow)
|
|
510
|
+
|
|
511
|
+
return {
|
|
512
|
+
"rps_now": rps_now,
|
|
513
|
+
"rps_1m": rps_1m,
|
|
514
|
+
"rps_10m": rps_10m,
|
|
515
|
+
"in_bps": in_bytes_per_s,
|
|
516
|
+
"out_bps": out_bytes_per_s,
|
|
517
|
+
"cpu_user": cpu_user,
|
|
518
|
+
"cpu_sys": cpu_sys,
|
|
519
|
+
"rss_total": rss_total,
|
|
520
|
+
"rss_max": rss_max,
|
|
521
|
+
"busy": busy,
|
|
522
|
+
"threads_total": threads_total,
|
|
523
|
+
"queue_mean": (queue_mean / queue_count) if queue_count else None,
|
|
524
|
+
"p50": p50, "p95": p95, "p99": p99,
|
|
525
|
+
"min": lo, "max": hi,
|
|
526
|
+
"active_slow": active_slow,
|
|
527
|
+
"recent_slow": recent_slow,
|
|
528
|
+
"total_slow": total_slow,
|
|
529
|
+
"status_1xx": status_1xx,
|
|
530
|
+
"status_2xx": status_2xx,
|
|
531
|
+
"status_3xx": status_3xx,
|
|
532
|
+
"status_4xx": status_4xx,
|
|
533
|
+
"status_5xx": status_5xx,
|
|
534
|
+
"interval_requests": interval_requests,
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def _avg_rps(state: State, ui: UIState, seconds: float) -> float:
|
|
539
|
+
total = 0
|
|
540
|
+
spans: list[float] = []
|
|
541
|
+
for ps in state.processes.values():
|
|
542
|
+
if ui.group_filter and ps.process_group != ui.group_filter:
|
|
543
|
+
continue
|
|
544
|
+
win = _samples_in_window(ps, seconds)
|
|
545
|
+
if len(win) < 2:
|
|
546
|
+
continue
|
|
547
|
+
oldest = win[-1].get("stamp", 0.0)
|
|
548
|
+
newest = win[0].get("stamp", 0.0)
|
|
549
|
+
span = newest - oldest
|
|
550
|
+
if span <= 0:
|
|
551
|
+
continue
|
|
552
|
+
rc = sum(int(s.get("fields", {}).get("request_count") or 0) for s in win)
|
|
553
|
+
total += rc
|
|
554
|
+
spans.append(span)
|
|
555
|
+
if not spans:
|
|
556
|
+
return 0.0
|
|
557
|
+
return total / (sum(spans) / len(spans))
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def render_header(win, state: State, ui: UIState, h: dict) -> int:
|
|
561
|
+
"""Draw the persistent header. Returns the row count consumed."""
|
|
562
|
+
width = win.getmaxyx()[1]
|
|
563
|
+
hostname = ""
|
|
564
|
+
pgroup = ui.group_filter or ""
|
|
565
|
+
sample_period = 0.0
|
|
566
|
+
if state.processes:
|
|
567
|
+
ps = next(iter(state.processes.values()))
|
|
568
|
+
hostname = ps.hostname
|
|
569
|
+
if not pgroup:
|
|
570
|
+
groups = _process_groups(state)
|
|
571
|
+
pgroup = ",".join(groups) if groups else "(any)"
|
|
572
|
+
sample_period = ps.sample_period
|
|
573
|
+
|
|
574
|
+
# Line 0 — title bar + connection state.
|
|
575
|
+
title = f" mod_wsgi-telemetry top {hostname or '-'} · {pgroup} "
|
|
576
|
+
status = "[paused]" if ui.paused else ("[live]" if state.connected else "[disconnected]")
|
|
577
|
+
extras = f" tick={sample_period:.1f}s refresh={ui.refresh:g}s win={_win_label(ui.window)} {status} "
|
|
578
|
+
pad = " " * max(0, width - len(title) - len(extras))
|
|
579
|
+
safe_addstr(win, 0, 0, title + pad + extras, curses.color_pair(CP_HEADER) | curses.A_BOLD)
|
|
580
|
+
|
|
581
|
+
# Line 1 — throughput. No padding on rps_now so the value starts
|
|
582
|
+
# at column 13, lining up vertically with the labels on the rows
|
|
583
|
+
# below (Capacity:, CPU:, Status:, Latency:, Slow:).
|
|
584
|
+
safe_addstr(win, 1, 0,
|
|
585
|
+
f" Throughput: {h['rps_now']:.1f} req/s now "
|
|
586
|
+
f"{h['rps_1m']:.1f}/s 1m {h['rps_10m']:.1f}/s 10m "
|
|
587
|
+
f"in {fmt_bytes(h['in_bps'])}/s out {fmt_bytes(h['out_bps'])}/s")
|
|
588
|
+
|
|
589
|
+
# Line 2 — capacity bar.
|
|
590
|
+
cap_total = h["threads_total"]
|
|
591
|
+
cap_busy = h["busy"]
|
|
592
|
+
frac = (cap_busy / cap_total) if cap_total else 0.0
|
|
593
|
+
bar_w = max(10, min(30, width - 60))
|
|
594
|
+
filled = int(round(frac * bar_w))
|
|
595
|
+
bar = BAR_FULL * filled + BAR_EMPTY * (bar_w - filled)
|
|
596
|
+
cap_attr = _cap_color(frac)
|
|
597
|
+
qstr = (f"queue {fmt_seconds(h['queue_mean'])}"
|
|
598
|
+
if h["queue_mean"] is not None else "queue -")
|
|
599
|
+
safe_addstr(win, 2, 0, " Capacity: ")
|
|
600
|
+
safe_addstr(win, 2, 13, bar, cap_attr)
|
|
601
|
+
safe_addstr(win, 2, 13 + bar_w + 1,
|
|
602
|
+
f" {cap_busy:>4d}/{cap_total:<4d} threads busy "
|
|
603
|
+
f"({frac*100:5.1f}%) {qstr}")
|
|
604
|
+
|
|
605
|
+
# Line 3 — CPU + memory.
|
|
606
|
+
safe_addstr(win, 3, 0,
|
|
607
|
+
f" CPU: user {h['cpu_user']:.2f} sys {h['cpu_sys']:.2f} "
|
|
608
|
+
f"total {h['cpu_user']+h['cpu_sys']:.2f} cores "
|
|
609
|
+
f"Memory: RSS {fmt_bytes(h['rss_total'])} (max {fmt_bytes(h['rss_max'])})")
|
|
610
|
+
|
|
611
|
+
# Line 4 — per-class HTTP response distribution for the latest
|
|
612
|
+
# interval. 2xx / 3xx / 4xx / 5xx as percentages of request_count.
|
|
613
|
+
# 1xx is a PEP 3333 tripwire (a WSGI app should never return 1xx)
|
|
614
|
+
# so it carries its raw count and renders amber when > 0; dim when
|
|
615
|
+
# zero. 5xx renders red-bold when > 0; 4xx is dim (mostly noise).
|
|
616
|
+
requests = h["interval_requests"]
|
|
617
|
+
def _pct(n):
|
|
618
|
+
return (n / requests * 100.0) if requests > 0 else 0.0
|
|
619
|
+
safe_addstr(win, 4, 0, " Status: ")
|
|
620
|
+
x = len(" Status: ")
|
|
621
|
+
s1 = f"1xx {h['status_1xx']}"
|
|
622
|
+
s1_attr = (curses.color_pair(CP_WARN) | curses.A_BOLD) if h["status_1xx"] > 0 \
|
|
623
|
+
else curses.color_pair(CP_DIM)
|
|
624
|
+
safe_addstr(win, 4, x, s1, s1_attr)
|
|
625
|
+
x += len(s1) + 2
|
|
626
|
+
s2 = f"2xx {_pct(h['status_2xx']):5.1f}%"
|
|
627
|
+
safe_addstr(win, 4, x, s2)
|
|
628
|
+
x += len(s2) + 2
|
|
629
|
+
s3 = f"3xx {_pct(h['status_3xx']):5.1f}%"
|
|
630
|
+
safe_addstr(win, 4, x, s3)
|
|
631
|
+
x += len(s3) + 2
|
|
632
|
+
s4 = f"4xx {_pct(h['status_4xx']):5.1f}%"
|
|
633
|
+
safe_addstr(win, 4, x, s4, curses.color_pair(CP_DIM))
|
|
634
|
+
x += len(s4) + 2
|
|
635
|
+
s5 = f"5xx {_pct(h['status_5xx']):5.1f}%"
|
|
636
|
+
s5_attr = (curses.color_pair(CP_CRIT) | curses.A_BOLD) if h["status_5xx"] > 0 \
|
|
637
|
+
else curses.color_pair(CP_DIM)
|
|
638
|
+
safe_addstr(win, 4, x, s5, s5_attr)
|
|
639
|
+
|
|
640
|
+
# Line 5 — latency percentiles.
|
|
641
|
+
p50 = fmt_seconds(h["p50"]); p95 = fmt_seconds(h["p95"]); p99 = fmt_seconds(h["p99"])
|
|
642
|
+
mn = fmt_seconds(h["min"]); mx = fmt_seconds(h["max"])
|
|
643
|
+
safe_addstr(win, 5, 0,
|
|
644
|
+
f" Latency: p50 {p50:>8} p95 {p95:>8} p99 {p99:>8} "
|
|
645
|
+
f"min {mn:>8} max {mx:>8}")
|
|
646
|
+
|
|
647
|
+
# Line 6 — slow-request counters.
|
|
648
|
+
safe_addstr(win, 6, 0,
|
|
649
|
+
f" Slow: {h['active_slow']} active / "
|
|
650
|
+
f"{h['recent_slow']} 1m / {h['total_slow']} total")
|
|
651
|
+
|
|
652
|
+
# Line 7 — tab bar.
|
|
653
|
+
_render_tabs(win, 7, width, ui.view)
|
|
654
|
+
return 8
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def _win_label(seconds: int) -> str:
|
|
658
|
+
if seconds < 60:
|
|
659
|
+
return f"{seconds}s"
|
|
660
|
+
if seconds < 3600:
|
|
661
|
+
return f"{seconds // 60}m"
|
|
662
|
+
return f"{seconds // 3600}h"
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
def _cap_color(frac: float) -> int:
|
|
666
|
+
if frac >= 0.9:
|
|
667
|
+
return curses.color_pair(CP_CRIT) | curses.A_BOLD
|
|
668
|
+
if frac >= 0.7:
|
|
669
|
+
return curses.color_pair(CP_WARN)
|
|
670
|
+
return curses.color_pair(CP_OK)
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def _render_tabs(win, y: int, width: int, active: str) -> None:
|
|
674
|
+
safe_addstr(win, y, 0, " " * width, curses.color_pair(CP_TAB))
|
|
675
|
+
x = 1
|
|
676
|
+
for i, name in enumerate(VIEWS, start=1):
|
|
677
|
+
label = f" {i} {name} "
|
|
678
|
+
attr = curses.color_pair(CP_TAB_ACTIVE) | curses.A_BOLD if name == active \
|
|
679
|
+
else curses.color_pair(CP_TAB)
|
|
680
|
+
safe_addstr(win, y, x, label, attr)
|
|
681
|
+
x += len(label)
|
|
682
|
+
if x >= width:
|
|
683
|
+
break
|
|
684
|
+
hint = " ?=help q=quit "
|
|
685
|
+
safe_addstr(win, y, max(x, width - len(hint)), hint, curses.color_pair(CP_TAB))
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
# --- Overview view -----------------------------------------------------------
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
def render_overview(win, state: State, ui: UIState, h: dict, y0: int) -> None:
|
|
692
|
+
height, width = win.getmaxyx()
|
|
693
|
+
avail = height - y0 - 1
|
|
694
|
+
if avail <= 0:
|
|
695
|
+
return
|
|
696
|
+
# Build a per-second time series for the window using the most-active pid
|
|
697
|
+
# as the time reference. Aggregate across pids on each second.
|
|
698
|
+
seconds = ui.window
|
|
699
|
+
rps_series, cap_series, cpu_series, rss_series = _build_series(state, ui, seconds)
|
|
700
|
+
|
|
701
|
+
spark_w = max(20, width - 18)
|
|
702
|
+
rows = [
|
|
703
|
+
("Throughput ", rps_series, lambda v: f"{v:.1f}/s now"),
|
|
704
|
+
("Capacity % ", cap_series, lambda v: f"{v*100:.0f}% now"),
|
|
705
|
+
("CPU cores ", cpu_series, lambda v: f"{v:.2f}/core"),
|
|
706
|
+
("RSS MB ", rss_series, lambda v: f"{v/1_048_576:.0f} MB"),
|
|
707
|
+
]
|
|
708
|
+
y = y0 + 1
|
|
709
|
+
safe_addstr(win, y0, 1, f"Overview — last {_win_label(seconds)} (use < > to change window)",
|
|
710
|
+
curses.A_BOLD)
|
|
711
|
+
for label, series, fmt in rows:
|
|
712
|
+
if y >= height - 1:
|
|
713
|
+
return
|
|
714
|
+
safe_addstr(win, y, 1, label)
|
|
715
|
+
line = sparkline(series, spark_w)
|
|
716
|
+
safe_addstr(win, y, 13, line)
|
|
717
|
+
if series:
|
|
718
|
+
safe_addstr(win, y, 14 + spark_w, fmt(series[-1]))
|
|
719
|
+
y += 1
|
|
720
|
+
|
|
721
|
+
y += 1
|
|
722
|
+
if y >= height - 1:
|
|
723
|
+
return
|
|
724
|
+
safe_addstr(win, y, 1, "Phase mean times (current tick):", curses.A_BOLD)
|
|
725
|
+
y += 1
|
|
726
|
+
if y >= height - 1:
|
|
727
|
+
return
|
|
728
|
+
parts = []
|
|
729
|
+
for phase in PHASES:
|
|
730
|
+
v = _aggregate_phase_mean(state, phase, ui.group_filter)
|
|
731
|
+
parts.append(f"{phase}={fmt_seconds(v) if v is not None else '-':>8}")
|
|
732
|
+
safe_addstr(win, y, 1, " ".join(parts))
|
|
733
|
+
|
|
734
|
+
|
|
735
|
+
def _build_series(state: State, ui: UIState, seconds: int):
|
|
736
|
+
"""Per-second aggregated series across pids for the last `seconds`."""
|
|
737
|
+
# Bucket samples by their second-stamp.
|
|
738
|
+
buckets: dict[int, dict[str, float]] = {}
|
|
739
|
+
for ps in state.processes.values():
|
|
740
|
+
if ui.group_filter and ps.process_group != ui.group_filter:
|
|
741
|
+
continue
|
|
742
|
+
for s in _samples_in_window(ps, seconds):
|
|
743
|
+
t = int(s.get("stamp", 0.0))
|
|
744
|
+
f = s.get("fields", {})
|
|
745
|
+
b = buckets.setdefault(t, {"rps": 0.0, "cap_b": 0, "cap_t": 0,
|
|
746
|
+
"cpu": 0.0, "rss": 0})
|
|
747
|
+
b["rps"] += float(f.get("request_throughput") or 0)
|
|
748
|
+
b["cap_b"] += int(f.get("request_threads_active") or 0)
|
|
749
|
+
b["cap_t"] += int(f.get("request_threads_maximum") or 0)
|
|
750
|
+
b["cpu"] += float(f.get("cpu_utilization") or 0)
|
|
751
|
+
b["rss"] += int(f.get("memory_rss") or 0)
|
|
752
|
+
keys = sorted(buckets)
|
|
753
|
+
rps = [buckets[k]["rps"] for k in keys]
|
|
754
|
+
cap = [(buckets[k]["cap_b"] / buckets[k]["cap_t"]) if buckets[k]["cap_t"] else 0.0
|
|
755
|
+
for k in keys]
|
|
756
|
+
cpu = [buckets[k]["cpu"] for k in keys]
|
|
757
|
+
rss = [buckets[k]["rss"] for k in keys]
|
|
758
|
+
return rps, cap, cpu, rss
|
|
759
|
+
|
|
760
|
+
|
|
761
|
+
def _aggregate_phase_mean(state: State, phase: str, group_filter: str | None):
|
|
762
|
+
"""Sample-weighted mean of last-tick `<phase>_time` across processes."""
|
|
763
|
+
name = f"{phase}_time"
|
|
764
|
+
total = 0.0
|
|
765
|
+
n = 0
|
|
766
|
+
for ps in state.processes.values():
|
|
767
|
+
if group_filter and ps.process_group != group_filter:
|
|
768
|
+
continue
|
|
769
|
+
if not ps.samples:
|
|
770
|
+
continue
|
|
771
|
+
v = ps.samples[-1].get("fields", {}).get(name)
|
|
772
|
+
if isinstance(v, (int, float)) and v > 0:
|
|
773
|
+
total += v
|
|
774
|
+
n += 1
|
|
775
|
+
return (total / n) if n else None
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
# --- Processes view ----------------------------------------------------------
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def render_processes(win, state: State, ui: UIState, y0: int) -> None:
|
|
782
|
+
height, width = win.getmaxyx()
|
|
783
|
+
rows = []
|
|
784
|
+
for ps in state.processes.values():
|
|
785
|
+
if ui.group_filter and ps.process_group != ui.group_filter:
|
|
786
|
+
continue
|
|
787
|
+
if not ps.samples:
|
|
788
|
+
continue
|
|
789
|
+
f = ps.samples[-1].get("fields", {})
|
|
790
|
+
buckets = []
|
|
791
|
+
for s in _samples_in_window(ps, 60):
|
|
792
|
+
b = s.get("fields", {}).get("request_time_buckets")
|
|
793
|
+
if isinstance(b, list) and len(b) == HDR_TOTAL:
|
|
794
|
+
if not buckets:
|
|
795
|
+
buckets = list(b)
|
|
796
|
+
else:
|
|
797
|
+
for i, c in enumerate(b):
|
|
798
|
+
buckets[i] += c
|
|
799
|
+
p95 = _hdr_percentile(buckets, 95) if buckets else None
|
|
800
|
+
slow_count = sum(1 for k, e in state.slow.items() if e.pid == ps.pid)
|
|
801
|
+
rows.append({
|
|
802
|
+
"pid": ps.pid,
|
|
803
|
+
"group": ps.process_group or "-",
|
|
804
|
+
"active": int(f.get("request_threads_active") or 0),
|
|
805
|
+
"max": int(f.get("request_threads_maximum") or 0),
|
|
806
|
+
"rps": float(f.get("request_throughput") or 0),
|
|
807
|
+
"cpu": float(f.get("cpu_utilization") or 0),
|
|
808
|
+
"rss": int(f.get("memory_rss") or 0),
|
|
809
|
+
"p95": p95,
|
|
810
|
+
"slow": slow_count,
|
|
811
|
+
})
|
|
812
|
+
|
|
813
|
+
sort_key = ui.process_sort
|
|
814
|
+
keyfn = {
|
|
815
|
+
"rps": lambda r: -r["rps"],
|
|
816
|
+
"cpu": lambda r: -r["cpu"],
|
|
817
|
+
"rss": lambda r: -r["rss"],
|
|
818
|
+
"p95": lambda r: -(r["p95"] or 0),
|
|
819
|
+
"slow": lambda r: -r["slow"],
|
|
820
|
+
"pid": lambda r: r["pid"],
|
|
821
|
+
}.get(sort_key, lambda r: -r["rps"])
|
|
822
|
+
rows.sort(key=keyfn)
|
|
823
|
+
|
|
824
|
+
safe_addstr(win, y0, 1,
|
|
825
|
+
f"Processes — sort: {sort_key} (use < > to change) "
|
|
826
|
+
f"{len(rows)} pids", curses.A_BOLD)
|
|
827
|
+
header = f" {'PID':>7} {'GROUP':<16} {'THREADS':>9} {'RPS':>7} " \
|
|
828
|
+
f"{'CPU':>6} {'RSS':>9} {'P95':>9} {'SLOW':>5}"
|
|
829
|
+
safe_addstr(win, y0 + 1, 0, header, curses.A_REVERSE)
|
|
830
|
+
y = y0 + 2
|
|
831
|
+
for r in rows:
|
|
832
|
+
if y >= height - 1:
|
|
833
|
+
break
|
|
834
|
+
line = (f" {r['pid']:>7} {r['group'][:16]:<16} "
|
|
835
|
+
f"{r['active']:>4}/{r['max']:<4} "
|
|
836
|
+
f"{r['rps']:>7.1f} "
|
|
837
|
+
f"{r['cpu']:>6.2f} "
|
|
838
|
+
f"{fmt_bytes(r['rss']):>9} "
|
|
839
|
+
f"{fmt_seconds(r['p95']):>9} "
|
|
840
|
+
f"{r['slow']:>5}")
|
|
841
|
+
attr = 0
|
|
842
|
+
if r["max"] and r["active"] / r["max"] >= 0.9:
|
|
843
|
+
attr = curses.color_pair(CP_CRIT)
|
|
844
|
+
elif r["max"] and r["active"] / r["max"] >= 0.7:
|
|
845
|
+
attr = curses.color_pair(CP_WARN)
|
|
846
|
+
safe_addstr(win, y, 0, line, attr)
|
|
847
|
+
y += 1
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
# --- Workers / slots view ----------------------------------------------------
|
|
851
|
+
|
|
852
|
+
|
|
853
|
+
def render_workers(win, state: State, ui: UIState, y0: int) -> None:
|
|
854
|
+
height, width = win.getmaxyx()
|
|
855
|
+
safe_addstr(win, y0, 1,
|
|
856
|
+
"Workers — slot grid: . idle * <1s # 1-5s ! >=slow-threshold",
|
|
857
|
+
curses.A_BOLD)
|
|
858
|
+
y = y0 + 1
|
|
859
|
+
pids = sorted(state.processes.keys())
|
|
860
|
+
for pid in pids:
|
|
861
|
+
if y >= height - 1:
|
|
862
|
+
break
|
|
863
|
+
ps = state.processes[pid]
|
|
864
|
+
if ui.group_filter and ps.process_group != ui.group_filter:
|
|
865
|
+
continue
|
|
866
|
+
if not ps.samples:
|
|
867
|
+
continue
|
|
868
|
+
last = ps.samples[-1].get("fields", {})
|
|
869
|
+
elapsed = last.get("request_threads_current_elapsed")
|
|
870
|
+
slow_thresh = last.get("slow_requests_threshold")
|
|
871
|
+
slow_thresh = (slow_thresh
|
|
872
|
+
if isinstance(slow_thresh, (int, float)) and slow_thresh > 0
|
|
873
|
+
else None)
|
|
874
|
+
if not isinstance(elapsed, list):
|
|
875
|
+
continue
|
|
876
|
+
cells = []
|
|
877
|
+
longest_idx = -1
|
|
878
|
+
longest = 0.0
|
|
879
|
+
for i, e in enumerate(elapsed):
|
|
880
|
+
if e <= 0:
|
|
881
|
+
cells.append(".")
|
|
882
|
+
elif slow_thresh is not None and e >= slow_thresh:
|
|
883
|
+
cells.append("!")
|
|
884
|
+
elif e >= 5.0:
|
|
885
|
+
cells.append("#")
|
|
886
|
+
elif e >= 1.0:
|
|
887
|
+
cells.append("#")
|
|
888
|
+
else:
|
|
889
|
+
cells.append("*")
|
|
890
|
+
if e > longest:
|
|
891
|
+
longest = e
|
|
892
|
+
longest_idx = i
|
|
893
|
+
|
|
894
|
+
label = f" pid {pid:<6} {ps.process_group[:14]:<14}"
|
|
895
|
+
safe_addstr(win, y, 0, label)
|
|
896
|
+
# Truncate slot row to terminal width.
|
|
897
|
+
avail = width - len(label) - 2
|
|
898
|
+
row = "".join(cells[:avail]) if avail > 0 else ""
|
|
899
|
+
# Color cells: red for !, yellow for #, default for *.
|
|
900
|
+
cx = len(label) + 1
|
|
901
|
+
for ch in row:
|
|
902
|
+
if cx >= width:
|
|
903
|
+
break
|
|
904
|
+
attr = 0
|
|
905
|
+
if ch == "!":
|
|
906
|
+
attr = curses.color_pair(CP_CRIT) | curses.A_BOLD
|
|
907
|
+
elif ch == "#":
|
|
908
|
+
attr = curses.color_pair(CP_WARN)
|
|
909
|
+
elif ch == "*":
|
|
910
|
+
attr = curses.color_pair(CP_OK)
|
|
911
|
+
else:
|
|
912
|
+
attr = curses.color_pair(CP_DIM)
|
|
913
|
+
safe_addstr(win, y, cx, ch, attr)
|
|
914
|
+
cx += 1
|
|
915
|
+
y += 1
|
|
916
|
+
if y >= height - 1:
|
|
917
|
+
break
|
|
918
|
+
if longest_idx >= 0 and longest > 0:
|
|
919
|
+
note = (f" longest slot: #{longest_idx} "
|
|
920
|
+
f"running {fmt_seconds(longest)}")
|
|
921
|
+
# Look for an active slow request matching this pid.
|
|
922
|
+
active = [e for e in state.slow.values()
|
|
923
|
+
if e.pid == pid and e.state == 0]
|
|
924
|
+
active.sort(key=lambda e: -e.duration)
|
|
925
|
+
if active:
|
|
926
|
+
e = active[0]
|
|
927
|
+
note += f" {e.method} {e.url()[:60]}"
|
|
928
|
+
safe_addstr(win, y, 0, note, curses.color_pair(CP_DIM))
|
|
929
|
+
y += 1
|
|
930
|
+
|
|
931
|
+
|
|
932
|
+
# --- Latency view ------------------------------------------------------------
|
|
933
|
+
|
|
934
|
+
|
|
935
|
+
def render_latency(win, state: State, ui: UIState, y0: int) -> None:
|
|
936
|
+
height, width = win.getmaxyx()
|
|
937
|
+
phase = ui.phase
|
|
938
|
+
seconds = ui.window
|
|
939
|
+
buckets = _aggregate_buckets(state, phase, seconds, ui.group_filter)
|
|
940
|
+
total = sum(buckets)
|
|
941
|
+
p50 = _hdr_percentile(buckets, 50)
|
|
942
|
+
p95 = _hdr_percentile(buckets, 95)
|
|
943
|
+
p99 = _hdr_percentile(buckets, 99)
|
|
944
|
+
lo, hi = _phase_min_max(state, phase, seconds, ui.group_filter)
|
|
945
|
+
|
|
946
|
+
safe_addstr(win, y0, 1,
|
|
947
|
+
f"Latency — phase: {phase} window: {_win_label(seconds)} "
|
|
948
|
+
f"samples: {total} "
|
|
949
|
+
f"([ ] phase, < > window)", curses.A_BOLD)
|
|
950
|
+
safe_addstr(win, y0 + 1, 1,
|
|
951
|
+
f" p50 {fmt_seconds(p50):>8} p95 {fmt_seconds(p95):>8} "
|
|
952
|
+
f"p99 {fmt_seconds(p99):>8} "
|
|
953
|
+
f"min {fmt_seconds(lo):>8} max {fmt_seconds(hi):>8}")
|
|
954
|
+
|
|
955
|
+
# Histogram. 65 buckets: render one column per bucket where it fits;
|
|
956
|
+
# otherwise pair them up so the chart still spans the whole window.
|
|
957
|
+
chart_top = y0 + 3
|
|
958
|
+
chart_bottom = height - 2
|
|
959
|
+
chart_h = max(4, chart_bottom - chart_top - 1)
|
|
960
|
+
chart_left = 4
|
|
961
|
+
chart_w = width - chart_left - 1
|
|
962
|
+
if chart_w <= 0 or chart_h <= 0:
|
|
963
|
+
return
|
|
964
|
+
|
|
965
|
+
cols_per_bucket = max(1, chart_w // HDR_TOTAL)
|
|
966
|
+
n_cols = HDR_TOTAL * cols_per_bucket
|
|
967
|
+
if n_cols > chart_w:
|
|
968
|
+
# Fallback: collapse buckets into chart_w cells by binning.
|
|
969
|
+
per_col = math.ceil(HDR_TOTAL / chart_w)
|
|
970
|
+
binned = []
|
|
971
|
+
for i in range(0, HDR_TOTAL, per_col):
|
|
972
|
+
binned.append(sum(buckets[i:i + per_col]))
|
|
973
|
+
display = binned
|
|
974
|
+
cols_per_bucket = 1
|
|
975
|
+
else:
|
|
976
|
+
display = []
|
|
977
|
+
for c in buckets:
|
|
978
|
+
display.extend([c] * cols_per_bucket)
|
|
979
|
+
|
|
980
|
+
vmax = max(display) if display else 0
|
|
981
|
+
if vmax <= 0:
|
|
982
|
+
safe_addstr(win, chart_top, chart_left, "(no samples in window)",
|
|
983
|
+
curses.color_pair(CP_DIM))
|
|
984
|
+
return
|
|
985
|
+
|
|
986
|
+
levels = chart_h * 8
|
|
987
|
+
# 8 sub-pixels per row using upper partial blocks.
|
|
988
|
+
PARTIAL = " ▁▂▃▄▅▆▇█"
|
|
989
|
+
# Render columns top-to-bottom.
|
|
990
|
+
for col, count in enumerate(display):
|
|
991
|
+
x = chart_left + col
|
|
992
|
+
if x >= width - 1:
|
|
993
|
+
break
|
|
994
|
+
h_units = int(round(count / vmax * levels))
|
|
995
|
+
full_rows = h_units // 8
|
|
996
|
+
rem = h_units % 8
|
|
997
|
+
# Bottom row anchored at chart_bottom.
|
|
998
|
+
for r in range(full_rows):
|
|
999
|
+
y = chart_bottom - r
|
|
1000
|
+
if y < chart_top:
|
|
1001
|
+
break
|
|
1002
|
+
safe_addstr(win, y, x, BAR_FULL, curses.color_pair(CP_BAR))
|
|
1003
|
+
if rem and full_rows < chart_h:
|
|
1004
|
+
y = chart_bottom - full_rows
|
|
1005
|
+
if y >= chart_top:
|
|
1006
|
+
safe_addstr(win, y, x, PARTIAL[rem], curses.color_pair(CP_BAR))
|
|
1007
|
+
|
|
1008
|
+
# Octave labels along the bottom.
|
|
1009
|
+
label_y = chart_bottom + 1
|
|
1010
|
+
if label_y < height:
|
|
1011
|
+
# Print a label every 4 buckets (= every octave start).
|
|
1012
|
+
for octave in range(0, HDR_OCTAVES, 2):
|
|
1013
|
+
lo = HDR_BASE_SECONDS * (1 << octave)
|
|
1014
|
+
x = chart_left + octave * HDR_SUBS * cols_per_bucket
|
|
1015
|
+
if x >= width:
|
|
1016
|
+
break
|
|
1017
|
+
safe_addstr(win, label_y, x, _short_seconds(lo),
|
|
1018
|
+
curses.color_pair(CP_DIM))
|
|
1019
|
+
|
|
1020
|
+
|
|
1021
|
+
def _short_seconds(s: float) -> str:
|
|
1022
|
+
"""Compact axis-tick label for a bucket boundary."""
|
|
1023
|
+
if s < 1.0:
|
|
1024
|
+
return f"{s*1000:.0f}ms"
|
|
1025
|
+
return f"{s:.0f}s"
|
|
1026
|
+
|
|
1027
|
+
|
|
1028
|
+
# --- Slow requests view ------------------------------------------------------
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
def render_slow(win, state: State, ui: UIState, y0: int) -> None:
|
|
1032
|
+
height, width = win.getmaxyx()
|
|
1033
|
+
rows = []
|
|
1034
|
+
now = time.time()
|
|
1035
|
+
for key, e in state.slow.items():
|
|
1036
|
+
if ui.slow_state_filter != -1 and e.state != ui.slow_state_filter:
|
|
1037
|
+
continue
|
|
1038
|
+
if ui.group_filter:
|
|
1039
|
+
ps = state.processes.get(e.pid)
|
|
1040
|
+
if not ps or ps.process_group != ui.group_filter:
|
|
1041
|
+
continue
|
|
1042
|
+
if ui.slow_search and ui.slow_search.lower() not in e.url().lower():
|
|
1043
|
+
continue
|
|
1044
|
+
if e.state == 0:
|
|
1045
|
+
elapsed = max(e.duration, now - e.start_stamp)
|
|
1046
|
+
else:
|
|
1047
|
+
elapsed = e.duration
|
|
1048
|
+
rows.append((elapsed, key, e))
|
|
1049
|
+
|
|
1050
|
+
sort_key = ui.slow_sort
|
|
1051
|
+
if sort_key == "elapsed":
|
|
1052
|
+
rows.sort(key=lambda r: -r[0])
|
|
1053
|
+
elif sort_key == "pid":
|
|
1054
|
+
rows.sort(key=lambda r: (r[2].pid, -r[0]))
|
|
1055
|
+
elif sort_key == "method":
|
|
1056
|
+
rows.sort(key=lambda r: (r[2].method, -r[0]))
|
|
1057
|
+
elif sort_key == "url":
|
|
1058
|
+
rows.sort(key=lambda r: (r[2].url(), -r[0]))
|
|
1059
|
+
|
|
1060
|
+
state_filter = {-1: "any", 0: "active", 1: "completed"}[ui.slow_state_filter]
|
|
1061
|
+
title = (f"Slow requests — sort: {sort_key} state: {state_filter} "
|
|
1062
|
+
f"({len(rows)} shown / {len(state.slow)} total) "
|
|
1063
|
+
f"(< > sort, f filter, / search)")
|
|
1064
|
+
safe_addstr(win, y0, 1, title, curses.A_BOLD)
|
|
1065
|
+
if ui.slow_search:
|
|
1066
|
+
safe_addstr(win, y0, len(title) + 2, f"search: {ui.slow_search!r}",
|
|
1067
|
+
curses.color_pair(CP_WARN))
|
|
1068
|
+
|
|
1069
|
+
header = f" {'STATE':<6} {'ELAPSED':>9} {'PID':>7} " \
|
|
1070
|
+
f"{'METHOD':<7} {'STATUS':>6} {'LOG ID':<24} URL"
|
|
1071
|
+
safe_addstr(win, y0 + 1, 0, header, curses.A_REVERSE)
|
|
1072
|
+
y = y0 + 2
|
|
1073
|
+
for elapsed, key, e in rows:
|
|
1074
|
+
if y >= height - 1:
|
|
1075
|
+
break
|
|
1076
|
+
st = "active" if e.state == 0 else "done"
|
|
1077
|
+
attr = curses.color_pair(CP_CRIT) | curses.A_BOLD if e.state == 0 \
|
|
1078
|
+
else curses.color_pair(CP_DIM)
|
|
1079
|
+
# status==0 means start_response wasn't called yet: show a dash
|
|
1080
|
+
# so it's visibly distinct from a real status code.
|
|
1081
|
+
status_str = "-" if not e.status else str(e.status)
|
|
1082
|
+
line = (f" {st:<6} {fmt_seconds(elapsed):>9} {e.pid:>7} "
|
|
1083
|
+
f"{e.method:<7} {status_str:>6} "
|
|
1084
|
+
f"{(e.log_id or '-')[:24]:<24} {e.url()}")
|
|
1085
|
+
safe_addstr(win, y, 0, line, attr)
|
|
1086
|
+
y += 1
|
|
1087
|
+
|
|
1088
|
+
if y < height - 1:
|
|
1089
|
+
# Show a brief I/O footer for the top entry.
|
|
1090
|
+
if rows:
|
|
1091
|
+
_, _, top = rows[0]
|
|
1092
|
+
footer = (f" top: cpu user={fmt_seconds(top.cpu_user_time)} "
|
|
1093
|
+
f"sys={fmt_seconds(top.cpu_system_time)} "
|
|
1094
|
+
f"i/o in={fmt_bytes(top.input_bytes)}/{top.input_reads}r "
|
|
1095
|
+
f"out={fmt_bytes(top.output_bytes)}/{top.output_writes}w")
|
|
1096
|
+
safe_addstr(win, height - 2, 0, footer, curses.color_pair(CP_DIM))
|
|
1097
|
+
|
|
1098
|
+
|
|
1099
|
+
# --- Help overlay ------------------------------------------------------------
|
|
1100
|
+
|
|
1101
|
+
|
|
1102
|
+
HELP_LINES = [
|
|
1103
|
+
"Keys:",
|
|
1104
|
+
" o / 1 overview p / 2 processes",
|
|
1105
|
+
" w / 3 workers l / 4 latency",
|
|
1106
|
+
" s / 5 slow requests",
|
|
1107
|
+
"",
|
|
1108
|
+
" space pause/resume rendering r reset/clear (slow view)",
|
|
1109
|
+
" + / - change refresh rate ? toggle this help",
|
|
1110
|
+
" < / > change sort or window [ / ] cycle phase (latency)",
|
|
1111
|
+
" f cycle filters (group, slow state)",
|
|
1112
|
+
" / slow URL search (slow view; type then Enter; Esc clears)",
|
|
1113
|
+
" q quit",
|
|
1114
|
+
]
|
|
1115
|
+
|
|
1116
|
+
|
|
1117
|
+
def render_help(win) -> None:
|
|
1118
|
+
h, w = win.getmaxyx()
|
|
1119
|
+
box_w = max(60, max(len(line) for line in HELP_LINES) + 4)
|
|
1120
|
+
box_h = len(HELP_LINES) + 4
|
|
1121
|
+
y0 = max(2, (h - box_h) // 2)
|
|
1122
|
+
x0 = max(2, (w - box_w) // 2)
|
|
1123
|
+
for r in range(box_h):
|
|
1124
|
+
safe_addstr(win, y0 + r, x0, " " * box_w, curses.color_pair(CP_TAB))
|
|
1125
|
+
safe_addstr(win, y0, x0, " " * box_w, curses.color_pair(CP_HEADER) | curses.A_BOLD)
|
|
1126
|
+
safe_addstr(win, y0, x0 + 2, " mod_wsgi-telemetry top — help ",
|
|
1127
|
+
curses.color_pair(CP_HEADER) | curses.A_BOLD)
|
|
1128
|
+
for i, line in enumerate(HELP_LINES):
|
|
1129
|
+
safe_addstr(win, y0 + 2 + i, x0 + 2, line, curses.color_pair(CP_TAB))
|
|
1130
|
+
|
|
1131
|
+
|
|
1132
|
+
# ---------------------------------------------------------------------------
|
|
1133
|
+
# WebSocket client
|
|
1134
|
+
# ---------------------------------------------------------------------------
|
|
1135
|
+
|
|
1136
|
+
|
|
1137
|
+
async def ws_client(url: str, state: State, stop: asyncio.Event) -> None:
|
|
1138
|
+
"""Connect, drain, reconnect on failure with backoff."""
|
|
1139
|
+
backoff = 1.0
|
|
1140
|
+
state.url = url
|
|
1141
|
+
while not stop.is_set():
|
|
1142
|
+
try:
|
|
1143
|
+
timeout = aiohttp.ClientTimeout(total=None, sock_connect=5, sock_read=None)
|
|
1144
|
+
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
1145
|
+
async with session.ws_connect(url, heartbeat=30) as ws:
|
|
1146
|
+
state.connected = True
|
|
1147
|
+
state.last_error = ""
|
|
1148
|
+
backoff = 1.0
|
|
1149
|
+
async for msg in ws:
|
|
1150
|
+
if msg.type == aiohttp.WSMsgType.TEXT:
|
|
1151
|
+
try:
|
|
1152
|
+
payload = json.loads(msg.data)
|
|
1153
|
+
except json.JSONDecodeError:
|
|
1154
|
+
continue
|
|
1155
|
+
state.apply(payload)
|
|
1156
|
+
elif msg.type in (aiohttp.WSMsgType.CLOSED,
|
|
1157
|
+
aiohttp.WSMsgType.ERROR):
|
|
1158
|
+
break
|
|
1159
|
+
if stop.is_set():
|
|
1160
|
+
break
|
|
1161
|
+
except asyncio.CancelledError:
|
|
1162
|
+
break
|
|
1163
|
+
except Exception as e:
|
|
1164
|
+
state.last_error = f"{type(e).__name__}: {e}"
|
|
1165
|
+
state.connected = False
|
|
1166
|
+
if stop.is_set():
|
|
1167
|
+
break
|
|
1168
|
+
try:
|
|
1169
|
+
await asyncio.wait_for(stop.wait(), timeout=backoff)
|
|
1170
|
+
break
|
|
1171
|
+
except asyncio.TimeoutError:
|
|
1172
|
+
pass
|
|
1173
|
+
backoff = min(backoff * 2, 10.0)
|
|
1174
|
+
|
|
1175
|
+
|
|
1176
|
+
# ---------------------------------------------------------------------------
|
|
1177
|
+
# Curses setup + dispatch
|
|
1178
|
+
# ---------------------------------------------------------------------------
|
|
1179
|
+
|
|
1180
|
+
|
|
1181
|
+
def init_colors() -> None:
|
|
1182
|
+
if not curses.has_colors():
|
|
1183
|
+
return
|
|
1184
|
+
curses.start_color()
|
|
1185
|
+
try:
|
|
1186
|
+
curses.use_default_colors()
|
|
1187
|
+
bg = -1
|
|
1188
|
+
except curses.error:
|
|
1189
|
+
bg = curses.COLOR_BLACK
|
|
1190
|
+
curses.init_pair(CP_HEADER, curses.COLOR_BLACK, curses.COLOR_CYAN)
|
|
1191
|
+
curses.init_pair(CP_OK, curses.COLOR_GREEN, bg)
|
|
1192
|
+
curses.init_pair(CP_WARN, curses.COLOR_YELLOW, bg)
|
|
1193
|
+
curses.init_pair(CP_CRIT, curses.COLOR_RED, bg)
|
|
1194
|
+
curses.init_pair(CP_DIM, curses.COLOR_WHITE, bg)
|
|
1195
|
+
curses.init_pair(CP_BAR, curses.COLOR_CYAN, bg)
|
|
1196
|
+
curses.init_pair(CP_TAB, curses.COLOR_WHITE, curses.COLOR_BLUE)
|
|
1197
|
+
curses.init_pair(CP_TAB_ACTIVE, curses.COLOR_BLACK, curses.COLOR_WHITE)
|
|
1198
|
+
|
|
1199
|
+
|
|
1200
|
+
class App:
|
|
1201
|
+
def __init__(self, stdscr, state: State, ui: UIState, stop: asyncio.Event,
|
|
1202
|
+
*, no_color: bool) -> None:
|
|
1203
|
+
self.stdscr = stdscr
|
|
1204
|
+
self.state = state
|
|
1205
|
+
self.ui = ui
|
|
1206
|
+
self.stop = stop
|
|
1207
|
+
self.no_color = no_color
|
|
1208
|
+
self.search_mode = False
|
|
1209
|
+
self.search_buf = ""
|
|
1210
|
+
|
|
1211
|
+
def handle_key(self, ch: int) -> None:
|
|
1212
|
+
ui = self.ui
|
|
1213
|
+
if self.search_mode:
|
|
1214
|
+
self._handle_search_key(ch)
|
|
1215
|
+
return
|
|
1216
|
+
if ch in (ord("q"), 27): # q or ESC
|
|
1217
|
+
self.stop.set()
|
|
1218
|
+
return
|
|
1219
|
+
if ch == ord("?") or ch == ord("h"):
|
|
1220
|
+
ui.show_help = not ui.show_help
|
|
1221
|
+
return
|
|
1222
|
+
if ch == ord(" "):
|
|
1223
|
+
ui.paused = not ui.paused
|
|
1224
|
+
return
|
|
1225
|
+
if ch == ord("+"):
|
|
1226
|
+
ui.refresh_idx = max(0, ui.refresh_idx - 1)
|
|
1227
|
+
return
|
|
1228
|
+
if ch == ord("-"):
|
|
1229
|
+
ui.refresh_idx = min(len(REFRESH_RATES) - 1, ui.refresh_idx + 1)
|
|
1230
|
+
return
|
|
1231
|
+
if ch in VIEW_KEYS:
|
|
1232
|
+
ui.view = VIEW_KEYS[ch]
|
|
1233
|
+
return
|
|
1234
|
+
if ch == ord("<") or ch == ord(","):
|
|
1235
|
+
self._cycle_left()
|
|
1236
|
+
return
|
|
1237
|
+
if ch == ord(">") or ch == ord("."):
|
|
1238
|
+
self._cycle_right()
|
|
1239
|
+
return
|
|
1240
|
+
if ch == ord("[") and ui.view == "latency":
|
|
1241
|
+
ui.phase_idx = (ui.phase_idx - 1) % len(PHASES)
|
|
1242
|
+
return
|
|
1243
|
+
if ch == ord("]") and ui.view == "latency":
|
|
1244
|
+
ui.phase_idx = (ui.phase_idx + 1) % len(PHASES)
|
|
1245
|
+
return
|
|
1246
|
+
if ch == ord("f"):
|
|
1247
|
+
self._cycle_filter()
|
|
1248
|
+
return
|
|
1249
|
+
if ch == ord("r") and ui.view == "slow":
|
|
1250
|
+
ui.slow_search = ""
|
|
1251
|
+
ui.slow_state_filter = -1
|
|
1252
|
+
return
|
|
1253
|
+
if ch == ord("/") and ui.view == "slow":
|
|
1254
|
+
self.search_mode = True
|
|
1255
|
+
self.search_buf = ui.slow_search
|
|
1256
|
+
return
|
|
1257
|
+
|
|
1258
|
+
def _handle_search_key(self, ch: int) -> None:
|
|
1259
|
+
if ch in (10, 13):
|
|
1260
|
+
self.ui.slow_search = self.search_buf
|
|
1261
|
+
self.search_mode = False
|
|
1262
|
+
elif ch == 27:
|
|
1263
|
+
self.search_mode = False
|
|
1264
|
+
self.search_buf = ""
|
|
1265
|
+
self.ui.slow_search = ""
|
|
1266
|
+
elif ch in (curses.KEY_BACKSPACE, 127, 8):
|
|
1267
|
+
self.search_buf = self.search_buf[:-1]
|
|
1268
|
+
elif 32 <= ch < 127:
|
|
1269
|
+
self.search_buf += chr(ch)
|
|
1270
|
+
|
|
1271
|
+
def _cycle_left(self) -> None:
|
|
1272
|
+
ui = self.ui
|
|
1273
|
+
if ui.view in ("overview", "latency"):
|
|
1274
|
+
ui.window_idx = (ui.window_idx - 1) % len(WINDOW_CHOICES)
|
|
1275
|
+
elif ui.view == "processes":
|
|
1276
|
+
keys = ["rps", "cpu", "rss", "p95", "slow", "pid"]
|
|
1277
|
+
i = keys.index(ui.process_sort) if ui.process_sort in keys else 0
|
|
1278
|
+
ui.process_sort = keys[(i - 1) % len(keys)]
|
|
1279
|
+
elif ui.view == "slow":
|
|
1280
|
+
keys = ["elapsed", "pid", "method", "url"]
|
|
1281
|
+
i = keys.index(ui.slow_sort) if ui.slow_sort in keys else 0
|
|
1282
|
+
ui.slow_sort = keys[(i - 1) % len(keys)]
|
|
1283
|
+
|
|
1284
|
+
def _cycle_right(self) -> None:
|
|
1285
|
+
ui = self.ui
|
|
1286
|
+
if ui.view in ("overview", "latency"):
|
|
1287
|
+
ui.window_idx = (ui.window_idx + 1) % len(WINDOW_CHOICES)
|
|
1288
|
+
elif ui.view == "processes":
|
|
1289
|
+
keys = ["rps", "cpu", "rss", "p95", "slow", "pid"]
|
|
1290
|
+
i = keys.index(ui.process_sort) if ui.process_sort in keys else 0
|
|
1291
|
+
ui.process_sort = keys[(i + 1) % len(keys)]
|
|
1292
|
+
elif ui.view == "slow":
|
|
1293
|
+
keys = ["elapsed", "pid", "method", "url"]
|
|
1294
|
+
i = keys.index(ui.slow_sort) if ui.slow_sort in keys else 0
|
|
1295
|
+
ui.slow_sort = keys[(i + 1) % len(keys)]
|
|
1296
|
+
|
|
1297
|
+
def _cycle_filter(self) -> None:
|
|
1298
|
+
ui = self.ui
|
|
1299
|
+
if ui.view == "slow":
|
|
1300
|
+
seq = [-1, 0, 1]
|
|
1301
|
+
i = seq.index(ui.slow_state_filter) if ui.slow_state_filter in seq else 0
|
|
1302
|
+
ui.slow_state_filter = seq[(i + 1) % len(seq)]
|
|
1303
|
+
return
|
|
1304
|
+
groups = _process_groups(self.state)
|
|
1305
|
+
if not groups:
|
|
1306
|
+
ui.group_filter = None
|
|
1307
|
+
return
|
|
1308
|
+
cycle = [None] + groups
|
|
1309
|
+
try:
|
|
1310
|
+
i = cycle.index(ui.group_filter)
|
|
1311
|
+
except ValueError:
|
|
1312
|
+
i = 0
|
|
1313
|
+
ui.group_filter = cycle[(i + 1) % len(cycle)]
|
|
1314
|
+
|
|
1315
|
+
def render(self) -> None:
|
|
1316
|
+
stdscr = self.stdscr
|
|
1317
|
+
stdscr.erase()
|
|
1318
|
+
h = _aggregate_header(self.state, self.ui)
|
|
1319
|
+
y0 = render_header(stdscr, self.state, self.ui, h)
|
|
1320
|
+
view = self.ui.view
|
|
1321
|
+
if view == "overview":
|
|
1322
|
+
render_overview(stdscr, self.state, self.ui, h, y0)
|
|
1323
|
+
elif view == "processes":
|
|
1324
|
+
render_processes(stdscr, self.state, self.ui, y0)
|
|
1325
|
+
elif view == "workers":
|
|
1326
|
+
render_workers(stdscr, self.state, self.ui, y0)
|
|
1327
|
+
elif view == "latency":
|
|
1328
|
+
render_latency(stdscr, self.state, self.ui, y0)
|
|
1329
|
+
elif view == "slow":
|
|
1330
|
+
render_slow(stdscr, self.state, self.ui, y0)
|
|
1331
|
+
|
|
1332
|
+
# Footer.
|
|
1333
|
+
height, width = stdscr.getmaxyx()
|
|
1334
|
+
footer = self._footer_text()
|
|
1335
|
+
safe_addstr(stdscr, height - 1, 0, footer.ljust(width)[:width],
|
|
1336
|
+
curses.color_pair(CP_TAB))
|
|
1337
|
+
if self.ui.show_help:
|
|
1338
|
+
render_help(stdscr)
|
|
1339
|
+
stdscr.noutrefresh()
|
|
1340
|
+
curses.doupdate()
|
|
1341
|
+
|
|
1342
|
+
def _footer_text(self) -> str:
|
|
1343
|
+
if self.search_mode:
|
|
1344
|
+
return f" / search: {self.search_buf}_ (Enter=apply Esc=cancel) "
|
|
1345
|
+
st = self.state
|
|
1346
|
+
if not st.connected:
|
|
1347
|
+
return f" disconnected ({st.last_error or 'connecting...'}) url={st.url} "
|
|
1348
|
+
age = time.monotonic() - st.last_message if st.last_message else 0
|
|
1349
|
+
return (f" connected to {st.url} pids={len(st.processes)} "
|
|
1350
|
+
f"slow={len(st.slow)} rcvd={st.total_received} "
|
|
1351
|
+
f"last_msg={age:.1f}s ago "
|
|
1352
|
+
f"{'PAUSED' if self.ui.paused else ''}")
|
|
1353
|
+
|
|
1354
|
+
|
|
1355
|
+
async def run_app(stdscr, args, state: State, ui: UIState) -> int:
|
|
1356
|
+
init_colors() if not args.no_color else None
|
|
1357
|
+
curses.curs_set(0)
|
|
1358
|
+
stdscr.nodelay(True)
|
|
1359
|
+
stdscr.keypad(True)
|
|
1360
|
+
|
|
1361
|
+
stop = asyncio.Event()
|
|
1362
|
+
loop = asyncio.get_running_loop()
|
|
1363
|
+
for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP):
|
|
1364
|
+
try:
|
|
1365
|
+
loop.add_signal_handler(sig, stop.set)
|
|
1366
|
+
except (NotImplementedError, ValueError):
|
|
1367
|
+
pass
|
|
1368
|
+
|
|
1369
|
+
ws_task = asyncio.create_task(ws_client(args.url, state, stop))
|
|
1370
|
+
app = App(stdscr, state, ui, stop, no_color=args.no_color)
|
|
1371
|
+
|
|
1372
|
+
try:
|
|
1373
|
+
last_render = 0.0
|
|
1374
|
+
while not stop.is_set():
|
|
1375
|
+
# Drain all pending keys.
|
|
1376
|
+
while True:
|
|
1377
|
+
ch = stdscr.getch()
|
|
1378
|
+
if ch == -1:
|
|
1379
|
+
break
|
|
1380
|
+
app.handle_key(ch)
|
|
1381
|
+
now = time.monotonic()
|
|
1382
|
+
if now - last_render >= ui.refresh and not ui.paused:
|
|
1383
|
+
app.render()
|
|
1384
|
+
last_render = now
|
|
1385
|
+
elif now - last_render >= ui.refresh:
|
|
1386
|
+
app.render()
|
|
1387
|
+
last_render = now
|
|
1388
|
+
await asyncio.sleep(0.05)
|
|
1389
|
+
finally:
|
|
1390
|
+
ws_task.cancel()
|
|
1391
|
+
try:
|
|
1392
|
+
await ws_task
|
|
1393
|
+
except (asyncio.CancelledError, Exception):
|
|
1394
|
+
pass
|
|
1395
|
+
|
|
1396
|
+
return 0
|
|
1397
|
+
|
|
1398
|
+
|
|
1399
|
+
def render_once_text(state: State, ui: UIState) -> str:
|
|
1400
|
+
"""Plain-text snapshot for --once. No curses, no colors."""
|
|
1401
|
+
h = _aggregate_header(state, ui)
|
|
1402
|
+
lines = []
|
|
1403
|
+
pgroup = ui.group_filter or ",".join(_process_groups(state)) or "(any)"
|
|
1404
|
+
hostname = next(iter(state.processes.values())).hostname if state.processes else "-"
|
|
1405
|
+
lines.append(f"mod_wsgi-telemetry top host={hostname} group={pgroup}")
|
|
1406
|
+
lines.append(f" rps: now {h['rps_now']:.1f} 1m {h['rps_1m']:.1f} "
|
|
1407
|
+
f"10m {h['rps_10m']:.1f} "
|
|
1408
|
+
f"in {fmt_bytes(h['in_bps'])}/s out {fmt_bytes(h['out_bps'])}/s")
|
|
1409
|
+
cap_total = h["threads_total"]
|
|
1410
|
+
cap_busy = h["busy"]
|
|
1411
|
+
frac = (cap_busy / cap_total) if cap_total else 0.0
|
|
1412
|
+
lines.append(f" cap: {cap_busy}/{cap_total} threads ({frac*100:.1f}%) "
|
|
1413
|
+
f"queue {fmt_seconds(h['queue_mean']) if h['queue_mean'] is not None else '-'}")
|
|
1414
|
+
lines.append(f" cpu: user {h['cpu_user']:.2f} sys {h['cpu_sys']:.2f} "
|
|
1415
|
+
f"rss {fmt_bytes(h['rss_total'])} (max {fmt_bytes(h['rss_max'])})")
|
|
1416
|
+
# status==0 (no start_response, exception path) is folded into 5xx
|
|
1417
|
+
# on the C side; the percentages here are share of completed
|
|
1418
|
+
# requests in the latest interval. 1xx is a PEP 3333 tripwire and
|
|
1419
|
+
# only included in the line when it has fired.
|
|
1420
|
+
requests = h["interval_requests"]
|
|
1421
|
+
pct = lambda n: (n / requests * 100.0) if requests > 0 else 0.0
|
|
1422
|
+
s_parts = []
|
|
1423
|
+
if h["status_1xx"] > 0:
|
|
1424
|
+
s_parts.append(f"1xx {h['status_1xx']} (PEP 3333 violation)")
|
|
1425
|
+
s_parts.append(f"2xx {pct(h['status_2xx']):.1f}%")
|
|
1426
|
+
s_parts.append(f"3xx {pct(h['status_3xx']):.1f}%")
|
|
1427
|
+
s_parts.append(f"4xx {pct(h['status_4xx']):.1f}%")
|
|
1428
|
+
s_parts.append(f"5xx {pct(h['status_5xx']):.1f}%")
|
|
1429
|
+
lines.append(" stat: " + " ".join(s_parts))
|
|
1430
|
+
lines.append(f" lat: p50 {fmt_seconds(h['p50'])} "
|
|
1431
|
+
f"p95 {fmt_seconds(h['p95'])} p99 {fmt_seconds(h['p99'])} "
|
|
1432
|
+
f"min {fmt_seconds(h['min'])} max {fmt_seconds(h['max'])}")
|
|
1433
|
+
lines.append(f" slow: {h['active_slow']} active / {h['recent_slow']} 1m / "
|
|
1434
|
+
f"{h['total_slow']} total")
|
|
1435
|
+
lines.append("")
|
|
1436
|
+
lines.append("processes:")
|
|
1437
|
+
rows = []
|
|
1438
|
+
for ps in state.processes.values():
|
|
1439
|
+
if ui.group_filter and ps.process_group != ui.group_filter:
|
|
1440
|
+
continue
|
|
1441
|
+
if not ps.samples:
|
|
1442
|
+
continue
|
|
1443
|
+
f = ps.samples[-1].get("fields", {})
|
|
1444
|
+
rows.append((
|
|
1445
|
+
ps.pid, ps.process_group or "-",
|
|
1446
|
+
int(f.get("request_threads_active") or 0),
|
|
1447
|
+
int(f.get("request_threads_maximum") or 0),
|
|
1448
|
+
float(f.get("request_throughput") or 0),
|
|
1449
|
+
float(f.get("cpu_utilization") or 0),
|
|
1450
|
+
int(f.get("memory_rss") or 0),
|
|
1451
|
+
))
|
|
1452
|
+
rows.sort(key=lambda r: -r[4])
|
|
1453
|
+
lines.append(f" {'PID':>7} {'GROUP':<16} {'THR':>9} {'RPS':>7} "
|
|
1454
|
+
f"{'CPU':>6} {'RSS':>9}")
|
|
1455
|
+
for pid, grp, act, mx, rps, cpu, rss in rows:
|
|
1456
|
+
lines.append(f" {pid:>7} {grp[:16]:<16} {act:>4}/{mx:<4} "
|
|
1457
|
+
f"{rps:>7.1f} {cpu:>6.2f} {fmt_bytes(rss):>9}")
|
|
1458
|
+
return "\n".join(lines)
|
|
1459
|
+
|
|
1460
|
+
|
|
1461
|
+
# ---------------------------------------------------------------------------
|
|
1462
|
+
# Entry point
|
|
1463
|
+
# ---------------------------------------------------------------------------
|
|
1464
|
+
|
|
1465
|
+
|
|
1466
|
+
def main(argv: list[str] | None = None) -> int:
|
|
1467
|
+
ap = argparse.ArgumentParser(
|
|
1468
|
+
description="Terminal monitor for mod_wsgi telemetry."
|
|
1469
|
+
)
|
|
1470
|
+
ap.add_argument("--url", default=os.environ.get("MOD_WSGI_TELEMETRY_URL", DEFAULT_URL),
|
|
1471
|
+
help=f"WebSocket URL of the telemetry server (default: {DEFAULT_URL})")
|
|
1472
|
+
ap.add_argument("--refresh", type=float, default=1.0,
|
|
1473
|
+
help="Render refresh interval in seconds (default: 1.0)")
|
|
1474
|
+
ap.add_argument("--view", choices=VIEWS, default="overview",
|
|
1475
|
+
help="Initial view (default: overview)")
|
|
1476
|
+
ap.add_argument("--group", default=None,
|
|
1477
|
+
help="Filter to a single process group")
|
|
1478
|
+
ap.add_argument("--no-color", action="store_true",
|
|
1479
|
+
help="Disable ANSI colour")
|
|
1480
|
+
ap.add_argument("--once", action="store_true",
|
|
1481
|
+
help="Render one frame to stdout and exit (scriptable)")
|
|
1482
|
+
ap.add_argument("--once-timeout", type=float, default=3.0,
|
|
1483
|
+
help="Max seconds to wait for the snapshot in --once mode")
|
|
1484
|
+
ap.add_argument("--log-file", default=None,
|
|
1485
|
+
help="Write debug log to this file (curses hides stderr)")
|
|
1486
|
+
args = ap.parse_args(argv)
|
|
1487
|
+
|
|
1488
|
+
if args.log_file:
|
|
1489
|
+
logging.basicConfig(filename=args.log_file, level=logging.DEBUG,
|
|
1490
|
+
format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
|
1491
|
+
|
|
1492
|
+
state = State()
|
|
1493
|
+
ui = UIState(view=args.view, group_filter=args.group)
|
|
1494
|
+
# Snap refresh to the closest preset so the cycler stays consistent.
|
|
1495
|
+
ui.refresh_idx = min(
|
|
1496
|
+
range(len(REFRESH_RATES)),
|
|
1497
|
+
key=lambda i: abs(REFRESH_RATES[i] - args.refresh),
|
|
1498
|
+
)
|
|
1499
|
+
|
|
1500
|
+
if args.once:
|
|
1501
|
+
return _run_once(args, state, ui)
|
|
1502
|
+
|
|
1503
|
+
return curses.wrapper(_curses_main, args, state, ui)
|
|
1504
|
+
|
|
1505
|
+
|
|
1506
|
+
def _curses_main(stdscr, args, state, ui) -> int:
|
|
1507
|
+
return asyncio.run(run_app(stdscr, args, state, ui))
|
|
1508
|
+
|
|
1509
|
+
|
|
1510
|
+
def _run_once(args, state: State, ui: UIState) -> int:
|
|
1511
|
+
async def grab() -> int:
|
|
1512
|
+
stop = asyncio.Event()
|
|
1513
|
+
ws_task = asyncio.create_task(ws_client(args.url, state, stop))
|
|
1514
|
+
deadline = time.monotonic() + args.once_timeout
|
|
1515
|
+
try:
|
|
1516
|
+
while time.monotonic() < deadline:
|
|
1517
|
+
if state.last_message:
|
|
1518
|
+
# Wait one extra tick to catch any in-flight live update.
|
|
1519
|
+
await asyncio.sleep(0.1)
|
|
1520
|
+
break
|
|
1521
|
+
await asyncio.sleep(0.05)
|
|
1522
|
+
sys.stdout.write(render_once_text(state, ui) + "\n")
|
|
1523
|
+
return 0 if state.last_message else 2
|
|
1524
|
+
finally:
|
|
1525
|
+
stop.set()
|
|
1526
|
+
ws_task.cancel()
|
|
1527
|
+
try:
|
|
1528
|
+
await ws_task
|
|
1529
|
+
except (asyncio.CancelledError, Exception):
|
|
1530
|
+
pass
|
|
1531
|
+
|
|
1532
|
+
return asyncio.run(grab())
|
|
1533
|
+
|
|
1534
|
+
|
|
1535
|
+
if __name__ == "__main__":
|
|
1536
|
+
sys.exit(main())
|