PyObservability 1.3.0__py3-none-any.whl → 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyobservability/config/settings.py +1 -0
- pyobservability/main.py +3 -3
- pyobservability/monitor.py +4 -5
- pyobservability/static/app.js +208 -8
- pyobservability/static/styles.css +112 -0
- pyobservability/templates/index.html +25 -0
- pyobservability/transport.py +151 -2
- pyobservability/version.py +1 -1
- {pyobservability-1.3.0.dist-info → pyobservability-1.4.0.dist-info}/METADATA +1 -1
- pyobservability-1.4.0.dist-info/RECORD +16 -0
- pyobservability-1.3.0.dist-info/RECORD +0 -16
- {pyobservability-1.3.0.dist-info → pyobservability-1.4.0.dist-info}/WHEEL +0 -0
- {pyobservability-1.3.0.dist-info → pyobservability-1.4.0.dist-info}/entry_points.txt +0 -0
- {pyobservability-1.3.0.dist-info → pyobservability-1.4.0.dist-info}/licenses/LICENSE +0 -0
- {pyobservability-1.3.0.dist-info → pyobservability-1.4.0.dist-info}/top_level.txt +0 -0
|
@@ -51,6 +51,7 @@ def detailed_log_config(filename: str | None = None, debug: bool = False) -> Dic
|
|
|
51
51
|
"uvicorn": {"handlers": ["default"], "level": level},
|
|
52
52
|
"uvicorn.error": {"handlers": ["default"], "level": level, "propagate": False},
|
|
53
53
|
"uvicorn.access": {"handlers": ["default"], "level": level, "propagate": False},
|
|
54
|
+
"uvicorn.default": {"handlers": ["default"], "level": level, "propagate": False},
|
|
54
55
|
},
|
|
55
56
|
"root": {"handlers": ["default"], "level": level},
|
|
56
57
|
}
|
pyobservability/main.py
CHANGED
|
@@ -2,7 +2,7 @@ import logging
|
|
|
2
2
|
import pathlib
|
|
3
3
|
import warnings
|
|
4
4
|
from datetime import datetime
|
|
5
|
-
from typing import Dict
|
|
5
|
+
from typing import Any, Dict
|
|
6
6
|
|
|
7
7
|
import uiauth
|
|
8
8
|
import uvicorn
|
|
@@ -39,7 +39,7 @@ async def index(request: Request):
|
|
|
39
39
|
TemplateResponse:
|
|
40
40
|
Rendered HTML template with targets and version.
|
|
41
41
|
"""
|
|
42
|
-
args = dict(request=request, targets=settings.env.targets, version=__version__)
|
|
42
|
+
args: Dict[str, Any] = dict(request=request, targets=settings.env.targets, version=__version__)
|
|
43
43
|
if settings.env.username and settings.env.password:
|
|
44
44
|
args["logout"] = uiauth.enums.APIEndpoints.fastapi_logout.value
|
|
45
45
|
return templates.TemplateResponse("index.html", args)
|
|
@@ -49,7 +49,7 @@ async def health() -> Dict[str, str]:
|
|
|
49
49
|
"""Health check endpoint.
|
|
50
50
|
|
|
51
51
|
Returns:
|
|
52
|
-
|
|
52
|
+
Dict[str, str]:
|
|
53
53
|
Health status.
|
|
54
54
|
"""
|
|
55
55
|
return {"status": "ok"}
|
pyobservability/monitor.py
CHANGED
|
@@ -174,7 +174,7 @@ class Monitor:
|
|
|
174
174
|
"ts": asyncio.get_event_loop().time(),
|
|
175
175
|
"data": [
|
|
176
176
|
{
|
|
177
|
-
"name": self.
|
|
177
|
+
"name": self.name,
|
|
178
178
|
"base_url": self.base_url,
|
|
179
179
|
"metrics": payload,
|
|
180
180
|
}
|
|
@@ -188,13 +188,12 @@ class Monitor:
|
|
|
188
188
|
_ = q.get_nowait()
|
|
189
189
|
q.put_nowait(result)
|
|
190
190
|
except Exception as err:
|
|
191
|
-
LOGGER.debug("Stream error for %s: %s", self.base_url, err)
|
|
192
191
|
if errors.get(self.base_url):
|
|
193
192
|
if errors[self.base_url] < 10:
|
|
193
|
+
LOGGER.debug("Stream error for %s: %s", self.base_url, err)
|
|
194
194
|
errors[self.base_url] += 1
|
|
195
195
|
else:
|
|
196
|
-
LOGGER.error(err
|
|
197
|
-
LOGGER.error("%s exceeded error threshold.", self.base_url)
|
|
196
|
+
LOGGER.error("Stream error for %s: %s", self.base_url, err)
|
|
198
197
|
|
|
199
198
|
# notify subscribers before stopping
|
|
200
199
|
error_msg = {
|
|
@@ -209,7 +208,7 @@ class Monitor:
|
|
|
209
208
|
except asyncio.QueueFull as warn:
|
|
210
209
|
LOGGER.warning(warn)
|
|
211
210
|
_ = q.get_nowait()
|
|
212
|
-
q.put_nowait(
|
|
211
|
+
q.put_nowait(error_msg)
|
|
213
212
|
await self.stop()
|
|
214
213
|
return
|
|
215
214
|
else:
|
pyobservability/static/app.js
CHANGED
|
@@ -370,6 +370,149 @@
|
|
|
370
370
|
const memChart = makeMainChart(memCtx, "Memory %");
|
|
371
371
|
const loadChart = makeMainChart(loadCtx, "CPU Load");
|
|
372
372
|
|
|
373
|
+
// Unified metrics: three parallel charts (memory, CPU, disk)
|
|
374
|
+
const unifiedPanel = document.getElementById("unified-panel");
|
|
375
|
+
const unifiedLegend = document.getElementById("unified-legend");
|
|
376
|
+
const unifiedMemCtx = document.getElementById("unified-mem-chart").getContext("2d");
|
|
377
|
+
const unifiedCpuCtx = document.getElementById("unified-cpu-chart").getContext("2d");
|
|
378
|
+
const unifiedDiskCtx = document.getElementById("unified-disk-chart").getContext("2d");
|
|
379
|
+
|
|
380
|
+
let unifiedNodes = [];
|
|
381
|
+
const colorPalette = ["#63b3ff", "#ff99c8", "#7dd3fc", "#fbbf24", "#a3e635", "#f87171", "#c084fc", "#38bdf8"];
|
|
382
|
+
const nodeColor = {};
|
|
383
|
+
const unifiedCharts = {memory: null, cpu: null, disk: null};
|
|
384
|
+
|
|
385
|
+
function normalizeNodes(nodes) {
|
|
386
|
+
return nodes
|
|
387
|
+
.filter(node => node.base_url && node.base_url !== "*")
|
|
388
|
+
.sort((a, b) => (a.name || a.base_url).localeCompare(b.name || b.base_url));
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
function assignColors(nodes) {
|
|
392
|
+
nodes.forEach((node, idx) => {
|
|
393
|
+
nodeColor[node.base_url] = colorPalette[idx % colorPalette.length];
|
|
394
|
+
});
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
function renderLegend(nodes) {
|
|
398
|
+
unifiedLegend.innerHTML = "";
|
|
399
|
+
nodes.forEach(node => {
|
|
400
|
+
const item = document.createElement("div");
|
|
401
|
+
item.className = "unified-legend-item";
|
|
402
|
+
item.innerHTML = `<span class="legend-dot" style="background:${nodeColor[node.base_url]}"></span>${node.name || node.base_url}`;
|
|
403
|
+
unifiedLegend.appendChild(item);
|
|
404
|
+
});
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
function makeUnifiedChart(ctx, nodes) {
|
|
408
|
+
return new Chart(ctx, {
|
|
409
|
+
type: "line",
|
|
410
|
+
data: {
|
|
411
|
+
labels: Array(MAX_POINTS).fill(""),
|
|
412
|
+
datasets: nodes.map(node => ({
|
|
413
|
+
label: node.name || node.base_url,
|
|
414
|
+
meta: {base_url: node.base_url},
|
|
415
|
+
data: Array(MAX_POINTS).fill(null),
|
|
416
|
+
borderColor: nodeColor[node.base_url],
|
|
417
|
+
backgroundColor: `${nodeColor[node.base_url]}33`,
|
|
418
|
+
borderWidth: 2,
|
|
419
|
+
fill: false,
|
|
420
|
+
tension: 0.3,
|
|
421
|
+
pointRadius: 0,
|
|
422
|
+
}))
|
|
423
|
+
},
|
|
424
|
+
options: {
|
|
425
|
+
animation: false,
|
|
426
|
+
responsive: true,
|
|
427
|
+
maintainAspectRatio: false,
|
|
428
|
+
scales: {
|
|
429
|
+
x: {display: false},
|
|
430
|
+
y: {beginAtZero: true, suggestedMax: 100}
|
|
431
|
+
},
|
|
432
|
+
plugins: {legend: {display: false}}
|
|
433
|
+
}
|
|
434
|
+
});
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
function setUnifiedMode(enabled) {
|
|
438
|
+
document.body.classList.toggle("unified-mode", enabled);
|
|
439
|
+
if (!enabled) {
|
|
440
|
+
unifiedPanel.classList.add("hidden");
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
function ensureUnifiedChart(metrics) {
|
|
445
|
+
const nodes = normalizeNodes(metrics);
|
|
446
|
+
if (!nodes.length) return false;
|
|
447
|
+
const changed =
|
|
448
|
+
nodes.length !== unifiedNodes.length ||
|
|
449
|
+
nodes.some((node, idx) => unifiedNodes[idx]?.base_url !== node.base_url);
|
|
450
|
+
|
|
451
|
+
if (changed) {
|
|
452
|
+
// Destroy any existing unified charts
|
|
453
|
+
Object.keys(unifiedCharts).forEach(key => {
|
|
454
|
+
if (unifiedCharts[key]) {
|
|
455
|
+
unifiedCharts[key].destroy();
|
|
456
|
+
unifiedCharts[key] = null;
|
|
457
|
+
}
|
|
458
|
+
});
|
|
459
|
+
assignColors(nodes);
|
|
460
|
+
unifiedCharts.memory = makeUnifiedChart(unifiedMemCtx, nodes);
|
|
461
|
+
unifiedCharts.cpu = makeUnifiedChart(unifiedCpuCtx, nodes);
|
|
462
|
+
unifiedCharts.disk = makeUnifiedChart(unifiedDiskCtx, nodes);
|
|
463
|
+
unifiedNodes = nodes;
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
unifiedPanel.classList.remove("hidden");
|
|
467
|
+
renderLegend(nodes);
|
|
468
|
+
return true;
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
function sampleForMetric(host, metric) {
|
|
472
|
+
if (!host.metrics) return null;
|
|
473
|
+
if (metric === "memory") return host.metrics.memory_info?.percent ?? null;
|
|
474
|
+
if (metric === "cpu") {
|
|
475
|
+
const values = (host.metrics.cpu_usage || [])
|
|
476
|
+
.map(v => Number(v))
|
|
477
|
+
.filter(Number.isFinite);
|
|
478
|
+
if (!values.length) return null;
|
|
479
|
+
return values.reduce((a, b) => a + b, 0) / values.length;
|
|
480
|
+
}
|
|
481
|
+
if (metric === "disk") {
|
|
482
|
+
const m = host.metrics;
|
|
483
|
+
if (m.disk_info && m.disk_info[0]) {
|
|
484
|
+
const m = host.metrics;
|
|
485
|
+
const agg = aggregateDiskInfo(m.disk_info);
|
|
486
|
+
return agg.percent;
|
|
487
|
+
}
|
|
488
|
+
return host.metrics.disk_info?.[0]?.percent ?? null;
|
|
489
|
+
}
|
|
490
|
+
return null;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
function updateUnified(metrics) {
|
|
494
|
+
const ts = new Date().toLocaleTimeString();
|
|
495
|
+
const metricKeys = ["memory", "cpu", "disk"];
|
|
496
|
+
metricKeys.forEach(metric => {
|
|
497
|
+
const chart = unifiedCharts[metric];
|
|
498
|
+
if (!chart) return;
|
|
499
|
+
|
|
500
|
+
chart.data.labels.push(ts);
|
|
501
|
+
if (chart.data.labels.length > MAX_POINTS) {
|
|
502
|
+
chart.data.labels.shift();
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
chart.data.datasets.forEach(ds => {
|
|
506
|
+
const host = metrics.find(h => h.base_url === ds.meta.base_url);
|
|
507
|
+
const value = host ? sampleForMetric(host, metric) : null;
|
|
508
|
+
ds.data.push(value);
|
|
509
|
+
if (ds.data.length > MAX_POINTS) ds.data.shift();
|
|
510
|
+
});
|
|
511
|
+
|
|
512
|
+
chart.update("none");
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
|
|
373
516
|
// ------------------------------------------------------------
|
|
374
517
|
// CORE CHARTS
|
|
375
518
|
// ------------------------------------------------------------
|
|
@@ -398,9 +541,9 @@
|
|
|
398
541
|
return coreMini[coreName] || createCoreChart(coreName);
|
|
399
542
|
}
|
|
400
543
|
|
|
401
|
-
function pruneOldCores(
|
|
544
|
+
function pruneOldCores(keep) {
|
|
402
545
|
for (const name of Object.keys(coreMini)) {
|
|
403
|
-
if (!
|
|
546
|
+
if (!keep.includes(name)) {
|
|
404
547
|
try {
|
|
405
548
|
coreMini[name].chart.destroy();
|
|
406
549
|
} catch {
|
|
@@ -440,11 +583,23 @@
|
|
|
440
583
|
resetChart(memChart);
|
|
441
584
|
resetChart(loadChart);
|
|
442
585
|
|
|
586
|
+
// Reset unified charts as well
|
|
587
|
+
Object.keys(unifiedCharts).forEach(key => {
|
|
588
|
+
const chart = unifiedCharts[key];
|
|
589
|
+
if (chart) {
|
|
590
|
+
try {
|
|
591
|
+
chart.destroy();
|
|
592
|
+
} catch {}
|
|
593
|
+
unifiedCharts[key] = null;
|
|
594
|
+
}
|
|
595
|
+
});
|
|
596
|
+
unifiedNodes = [];
|
|
597
|
+
unifiedPanel.classList.add("hidden");
|
|
598
|
+
|
|
443
599
|
for (const name of Object.keys(coreMini)) {
|
|
444
600
|
try {
|
|
445
601
|
coreMini[name].chart.destroy();
|
|
446
|
-
} catch {
|
|
447
|
-
}
|
|
602
|
+
} catch {}
|
|
448
603
|
coreMini[name].el.remove();
|
|
449
604
|
delete coreMini[name];
|
|
450
605
|
}
|
|
@@ -460,6 +615,19 @@
|
|
|
460
615
|
// ------------------------------------------------------------
|
|
461
616
|
// MISC HELPERS
|
|
462
617
|
// ------------------------------------------------------------
|
|
618
|
+
function aggregateDiskInfo(diskInfo) {
|
|
619
|
+
let totalDisk = 0;
|
|
620
|
+
let usedDisk = 0;
|
|
621
|
+
let freeDisk = 0;
|
|
622
|
+
diskInfo.forEach(d => {
|
|
623
|
+
totalDisk += num(d.total || 0);
|
|
624
|
+
usedDisk += num(d.used || 0);
|
|
625
|
+
freeDisk += num(d.free || 0);
|
|
626
|
+
});
|
|
627
|
+
let percentDisk = totalDisk > 0 ? (usedDisk / totalDisk) * 100 : 0.0;
|
|
628
|
+
return {"total": totalDisk, "used": usedDisk, "free": freeDisk, "percent": percentDisk};
|
|
629
|
+
}
|
|
630
|
+
|
|
463
631
|
function pushPoint(chart, value) {
|
|
464
632
|
const ts = new Date().toLocaleTimeString();
|
|
465
633
|
chart.data.labels.push(ts);
|
|
@@ -519,6 +687,16 @@
|
|
|
519
687
|
|
|
520
688
|
const now = new Date().toLocaleTimeString();
|
|
521
689
|
|
|
690
|
+
if (selectedBase === "*") {
|
|
691
|
+
if (ensureUnifiedChart(list)) {
|
|
692
|
+
setUnifiedMode(true);
|
|
693
|
+
updateUnified(list);
|
|
694
|
+
}
|
|
695
|
+
return;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
setUnifiedMode(false);
|
|
699
|
+
|
|
522
700
|
for (const host of list) {
|
|
523
701
|
if (host.base_url !== selectedBase) continue;
|
|
524
702
|
const m = host.metrics || {};
|
|
@@ -541,17 +719,19 @@
|
|
|
541
719
|
`GPU: ${m.gpu_name || "-"}`;
|
|
542
720
|
|
|
543
721
|
if (m.disk_info && m.disk_info[0]) {
|
|
544
|
-
const
|
|
722
|
+
const agg = aggregateDiskInfo(m.disk_info);
|
|
545
723
|
diskEl.textContent =
|
|
546
|
-
`Total: ${formatBytes(
|
|
547
|
-
`Used: ${formatBytes(
|
|
548
|
-
`Free: ${formatBytes(
|
|
724
|
+
`Total: ${formatBytes(agg.total)}\n` +
|
|
725
|
+
`Used: ${formatBytes(agg.used)}\n` +
|
|
726
|
+
`Free: ${formatBytes(agg.free)}\n` +
|
|
727
|
+
`Percent: ${round2(agg.percent)}%`;
|
|
549
728
|
}
|
|
550
729
|
|
|
551
730
|
if (m.memory_info) {
|
|
552
731
|
memEl.textContent =
|
|
553
732
|
`Total: ${formatBytes(m.memory_info.total)}\n` +
|
|
554
733
|
`Used: ${formatBytes(m.memory_info.used)}\n` +
|
|
734
|
+
`Free: ${formatBytes(m.memory_info.free)}\n` +
|
|
555
735
|
`Percent: ${round2(m.memory_info.percent)}%`;
|
|
556
736
|
pushPoint(memChart, num(m.memory_info.percent));
|
|
557
737
|
}
|
|
@@ -644,11 +824,30 @@
|
|
|
644
824
|
hideSpinner("certificates-table");
|
|
645
825
|
}
|
|
646
826
|
}
|
|
827
|
+
|
|
828
|
+
if (selectedBase === "*") {
|
|
829
|
+
if (ensureUnifiedChart(list)) {
|
|
830
|
+
updateUnified(list);
|
|
831
|
+
}
|
|
832
|
+
} else {
|
|
833
|
+
unifiedPanel.classList.add("hidden");
|
|
834
|
+
unifiedNodes = [];
|
|
835
|
+
Object.keys(unifiedCharts).forEach(key => {
|
|
836
|
+
if (unifiedCharts[key]) {
|
|
837
|
+
unifiedCharts[key].destroy();
|
|
838
|
+
unifiedCharts[key] = null;
|
|
839
|
+
}
|
|
840
|
+
});
|
|
841
|
+
}
|
|
647
842
|
}
|
|
648
843
|
|
|
649
844
|
// ------------------------------------------------------------
|
|
650
845
|
// EVENT BINDINGS
|
|
651
846
|
// ------------------------------------------------------------
|
|
847
|
+
targets.push({
|
|
848
|
+
base_url: "*",
|
|
849
|
+
name: "*"
|
|
850
|
+
});
|
|
652
851
|
targets.forEach(t => {
|
|
653
852
|
const opt = document.createElement("option");
|
|
654
853
|
opt.value = t.base_url;
|
|
@@ -664,6 +863,7 @@
|
|
|
664
863
|
resetUI();
|
|
665
864
|
resetTables();
|
|
666
865
|
showAllSpinners();
|
|
866
|
+
if (selectedBase !== "*") unifiedPanel.classList.add("hidden");
|
|
667
867
|
ws.send(JSON.stringify({type: "select_target", base_url: selectedBase}));
|
|
668
868
|
});
|
|
669
869
|
|
|
@@ -265,6 +265,10 @@ input#proc-filter {
|
|
|
265
265
|
.meta-row {
|
|
266
266
|
grid-template-columns: 1fr;
|
|
267
267
|
}
|
|
268
|
+
|
|
269
|
+
.unified-grid {
|
|
270
|
+
grid-template-columns: 1fr;
|
|
271
|
+
}
|
|
268
272
|
}
|
|
269
273
|
|
|
270
274
|
/* ---------------- SPINNER ------------------- */
|
|
@@ -324,3 +328,111 @@ input#proc-filter {
|
|
|
324
328
|
margin-bottom: 4px;
|
|
325
329
|
opacity: 0.85;
|
|
326
330
|
}
|
|
331
|
+
|
|
332
|
+
.panel-helper {
|
|
333
|
+
margin: 0 0 8px;
|
|
334
|
+
font-size: 12px;
|
|
335
|
+
color: var(--muted);
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
.metric-switch {
|
|
339
|
+
display: inline-flex;
|
|
340
|
+
border: 1px solid rgba(255, 255, 255, 0.08);
|
|
341
|
+
border-radius: 4px;
|
|
342
|
+
overflow: hidden;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
.metric-btn {
|
|
346
|
+
background: transparent;
|
|
347
|
+
color: inherit;
|
|
348
|
+
border: none;
|
|
349
|
+
padding: 6px 12px;
|
|
350
|
+
font-size: 12px;
|
|
351
|
+
cursor: pointer;
|
|
352
|
+
transition: background 0.2s, color 0.2s;
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
.metric-btn:not(.active):hover {
|
|
356
|
+
background: rgba(255, 255, 255, 0.05);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
.metric-btn.active {
|
|
360
|
+
background: var(--accent);
|
|
361
|
+
color: #020617;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
.unified-header {
|
|
365
|
+
align-items: center;
|
|
366
|
+
justify-content: center;
|
|
367
|
+
gap: 12px;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
.unified-title-group {
|
|
371
|
+
display: flex;
|
|
372
|
+
flex-direction: column;
|
|
373
|
+
align-items: center;
|
|
374
|
+
justify-content: center;
|
|
375
|
+
text-align: center;
|
|
376
|
+
gap: 2px;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
.unified-indicator {
|
|
380
|
+
font-size: 12px;
|
|
381
|
+
text-transform: uppercase;
|
|
382
|
+
letter-spacing: 0.05em;
|
|
383
|
+
color: var(--muted);
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
.unified-legend {
|
|
387
|
+
display: flex;
|
|
388
|
+
flex-wrap: wrap;
|
|
389
|
+
gap: 8px;
|
|
390
|
+
margin-top: 10px;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
.unified-legend-item {
|
|
394
|
+
display: flex;
|
|
395
|
+
align-items: center;
|
|
396
|
+
gap: 6px;
|
|
397
|
+
padding: 4px 8px;
|
|
398
|
+
border-radius: 4px;
|
|
399
|
+
background: rgba(255, 255, 255, 0.05);
|
|
400
|
+
font-size: 12px;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
.legend-dot {
|
|
404
|
+
width: 10px;
|
|
405
|
+
height: 10px;
|
|
406
|
+
border-radius: 50%;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/* New unified-grid layout for three unified charts */
|
|
410
|
+
.unified-grid {
|
|
411
|
+
display: grid;
|
|
412
|
+
grid-template-columns: repeat(3, minmax(0, 1fr));
|
|
413
|
+
gap: 12px;
|
|
414
|
+
margin-top: 8px;
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
.unified-metric-panel {
|
|
418
|
+
background: var(--panel);
|
|
419
|
+
border-radius: 8px;
|
|
420
|
+
padding: 8px 8px 4px;
|
|
421
|
+
box-shadow: 0 4px 12px rgba(2, 6, 23, 0.6);
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
.unified-metric-panel .panel-header h3 {
|
|
425
|
+
font-size: 13px;
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
body.unified-mode .meta-row,
|
|
429
|
+
body.unified-mode .charts-row > :not(#unified-panel),
|
|
430
|
+
body.unified-mode .details-row,
|
|
431
|
+
body.unified-mode .tables-row,
|
|
432
|
+
body.unified-mode footer {
|
|
433
|
+
display: none !important;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
body.unified-mode #unified-panel {
|
|
437
|
+
flex: 1;
|
|
438
|
+
}
|
|
@@ -75,6 +75,31 @@
|
|
|
75
75
|
<div class="panel-header"><h3>Memory (%)</h3></div>
|
|
76
76
|
<canvas id="mem-chart" class="chart"></canvas>
|
|
77
77
|
</div>
|
|
78
|
+
|
|
79
|
+
<div class="panel hidden" id="unified-panel">
|
|
80
|
+
<div class="panel-header unified-header">
|
|
81
|
+
<div class="unified-title-group">
|
|
82
|
+
<h3>Aggregated Metrics</h3>
|
|
83
|
+
<span class="unified-indicator">Holistic node comparison</span>
|
|
84
|
+
</div>
|
|
85
|
+
</div>
|
|
86
|
+
{# <p class="panel-helper">Select "All nodes" (*) to compare every host.</p> #}
|
|
87
|
+
<div class="unified-grid">
|
|
88
|
+
<div class="unified-metric-panel">
|
|
89
|
+
<div class="panel-header"><h3>Memory (%)</h3></div>
|
|
90
|
+
<canvas id="unified-mem-chart" class="chart"></canvas>
|
|
91
|
+
</div>
|
|
92
|
+
<div class="unified-metric-panel">
|
|
93
|
+
<div class="panel-header"><h3>CPU Avg (%)</h3></div>
|
|
94
|
+
<canvas id="unified-cpu-chart" class="chart"></canvas>
|
|
95
|
+
</div>
|
|
96
|
+
<div class="unified-metric-panel">
|
|
97
|
+
<div class="panel-header"><h3>Disk (%)</h3></div>
|
|
98
|
+
<canvas id="unified-disk-chart" class="chart"></canvas>
|
|
99
|
+
</div>
|
|
100
|
+
</div>
|
|
101
|
+
<div id="unified-legend" class="unified-legend"></div>
|
|
102
|
+
</div>
|
|
78
103
|
</section>
|
|
79
104
|
|
|
80
105
|
<section class="details-row">
|
pyobservability/transport.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import json
|
|
3
3
|
import logging
|
|
4
|
+
from typing import Dict, List
|
|
4
5
|
|
|
5
6
|
from fastapi import WebSocket, WebSocketDisconnect
|
|
6
7
|
|
|
@@ -22,6 +23,99 @@ async def _forward_metrics(websocket: WebSocket, q: asyncio.Queue) -> None:
|
|
|
22
23
|
await websocket.send_json(payload)
|
|
23
24
|
|
|
24
25
|
|
|
26
|
+
def _normalize_targets() -> List[Dict[str, str]]:
|
|
27
|
+
"""Return configuration targets sorted so legend colors are consistent."""
|
|
28
|
+
return sorted(settings.env.targets, key=lambda t: t["name"].lower())
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
async def _forward_metrics_multi(
|
|
32
|
+
websocket: WebSocket,
|
|
33
|
+
queues: List[asyncio.Queue],
|
|
34
|
+
targets: List[Dict[str, str]],
|
|
35
|
+
) -> None:
|
|
36
|
+
"""Fan-in metrics from multiple monitors and emit once every node updates.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
websocket: FastAPI WebSocket connection.
|
|
40
|
+
queues: List of asyncio.Queues from multiple monitors.
|
|
41
|
+
targets: List of target configurations corresponding to the queues.
|
|
42
|
+
|
|
43
|
+
Notes:
|
|
44
|
+
This is resilient to individual node failures.
|
|
45
|
+
If a monitor starts emitting an ``error`` payload (e.g. exceeded error threshold),
|
|
46
|
+
that node is marked as failed and removed from the unified stream,
|
|
47
|
+
so that the remaining nodes keep updating in the UI.
|
|
48
|
+
"""
|
|
49
|
+
# Track active (non-failed) indices
|
|
50
|
+
active = [True] * len(queues)
|
|
51
|
+
|
|
52
|
+
# Latest payload per node keyed by base_url
|
|
53
|
+
latest: Dict[str, dict] = {}
|
|
54
|
+
try:
|
|
55
|
+
while True:
|
|
56
|
+
# If all nodes have failed, stop the unified stream loop.
|
|
57
|
+
if not any(active):
|
|
58
|
+
LOGGER.info("All targets in unified stream have failed; stopping multi-node forwarder")
|
|
59
|
+
return
|
|
60
|
+
|
|
61
|
+
# Wait for at least one payload from any active queue.
|
|
62
|
+
# We gather one payload per active queue in a "round", but we do
|
|
63
|
+
# not block forever on failed queues because we mark them inactive
|
|
64
|
+
# as soon as they send an error.
|
|
65
|
+
for idx, q in enumerate(queues):
|
|
66
|
+
if not active[idx]:
|
|
67
|
+
continue
|
|
68
|
+
|
|
69
|
+
payload = await q.get()
|
|
70
|
+
|
|
71
|
+
# Handle error payloads from monitors: mark this node as failed
|
|
72
|
+
# and notify the UI once, then skip it from future
|
|
73
|
+
# rounds so the rest of the nodes continue streaming.
|
|
74
|
+
if isinstance(payload, dict) and payload.get("type") == "error":
|
|
75
|
+
base_url = targets[idx]["base_url"]
|
|
76
|
+
LOGGER.warning("Unified stream: target %s reported error and will be skipped", base_url)
|
|
77
|
+
|
|
78
|
+
# Forward the error to the websocket so the UI can show it.
|
|
79
|
+
try:
|
|
80
|
+
await websocket.send_json(payload)
|
|
81
|
+
except Exception as send_err: # pragma: no cover - defensive
|
|
82
|
+
LOGGER.debug("Failed to send error payload to WS: %s", send_err)
|
|
83
|
+
|
|
84
|
+
# Mark this target as inactive and remove its latest sample
|
|
85
|
+
active[idx] = False
|
|
86
|
+
latest.pop(base_url, None)
|
|
87
|
+
continue
|
|
88
|
+
|
|
89
|
+
# Normal metrics payload: remember latest per base_url
|
|
90
|
+
base_url = targets[idx]["base_url"]
|
|
91
|
+
latest[base_url] = payload
|
|
92
|
+
|
|
93
|
+
# Build merged message from all currently active targets that have
|
|
94
|
+
# produced at least one metrics payload.
|
|
95
|
+
merged = {
|
|
96
|
+
"type": "metrics",
|
|
97
|
+
"ts": asyncio.get_event_loop().time(),
|
|
98
|
+
"data": [],
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
for idx, target in enumerate(targets):
|
|
102
|
+
if not active[idx]:
|
|
103
|
+
continue
|
|
104
|
+
sample = latest.get(target["base_url"])
|
|
105
|
+
if not sample:
|
|
106
|
+
continue
|
|
107
|
+
merged["data"].extend(sample.get("data", []))
|
|
108
|
+
|
|
109
|
+
# If we have no data (e.g. new round before any active node
|
|
110
|
+
# produced metrics), skip sending to avoid spamming empty payloads.
|
|
111
|
+
if not merged["data"]:
|
|
112
|
+
continue
|
|
113
|
+
|
|
114
|
+
await websocket.send_json(merged)
|
|
115
|
+
except asyncio.CancelledError:
|
|
116
|
+
LOGGER.debug("Unified stream task cancelled")
|
|
117
|
+
|
|
118
|
+
|
|
25
119
|
async def websocket_endpoint(websocket: WebSocket) -> None:
|
|
26
120
|
"""Websocket endpoint to handle observability data streaming.
|
|
27
121
|
|
|
@@ -32,6 +126,10 @@ async def websocket_endpoint(websocket: WebSocket) -> None:
|
|
|
32
126
|
|
|
33
127
|
monitor: Monitor | None = None
|
|
34
128
|
q: asyncio.Queue | None = None
|
|
129
|
+
monitors: List[Monitor] = []
|
|
130
|
+
queues: List[asyncio.Queue] = []
|
|
131
|
+
multi_task: asyncio.Task | None = None
|
|
132
|
+
forward_task: asyncio.Task | None = None
|
|
35
133
|
|
|
36
134
|
try:
|
|
37
135
|
while True:
|
|
@@ -42,6 +140,15 @@ async def websocket_endpoint(websocket: WebSocket) -> None:
|
|
|
42
140
|
await monitor.update_flags(
|
|
43
141
|
all_services=data.get("all_services", False),
|
|
44
142
|
)
|
|
143
|
+
elif monitors:
|
|
144
|
+
await asyncio.gather(
|
|
145
|
+
*(
|
|
146
|
+
mon.update_flags(
|
|
147
|
+
all_services=data.get("all_services", False),
|
|
148
|
+
)
|
|
149
|
+
for mon in monitors
|
|
150
|
+
)
|
|
151
|
+
)
|
|
45
152
|
continue
|
|
46
153
|
|
|
47
154
|
# -------------------------------------------
|
|
@@ -50,10 +157,42 @@ async def websocket_endpoint(websocket: WebSocket) -> None:
|
|
|
50
157
|
if data.get("type") == "select_target":
|
|
51
158
|
base_url = data["base_url"]
|
|
52
159
|
|
|
53
|
-
# stop old monitor
|
|
160
|
+
# stop old monitor / tasks
|
|
161
|
+
if forward_task:
|
|
162
|
+
LOGGER.info("Stopping previous forwarder task")
|
|
163
|
+
forward_task.cancel()
|
|
164
|
+
forward_task = None
|
|
165
|
+
|
|
54
166
|
if monitor:
|
|
167
|
+
LOGGER.info("Stopping previous monitor task")
|
|
55
168
|
monitor.unsubscribe(q)
|
|
56
169
|
await monitor.stop()
|
|
170
|
+
monitor = None
|
|
171
|
+
q = None
|
|
172
|
+
|
|
173
|
+
if multi_task:
|
|
174
|
+
LOGGER.info("Stopping previous multi-target forwarder task")
|
|
175
|
+
multi_task.cancel()
|
|
176
|
+
multi_task = None
|
|
177
|
+
|
|
178
|
+
LOGGER.info("Unsubscribing from previous monitors") if monitor else None
|
|
179
|
+
for idx, mon in enumerate(monitors):
|
|
180
|
+
mon.unsubscribe(queues[idx])
|
|
181
|
+
await mon.stop()
|
|
182
|
+
|
|
183
|
+
monitors.clear()
|
|
184
|
+
queues.clear()
|
|
185
|
+
|
|
186
|
+
if base_url == "*":
|
|
187
|
+
LOGGER.info("Gathering metrics for all targets in unified stream")
|
|
188
|
+
targets = _normalize_targets()
|
|
189
|
+
for target in targets:
|
|
190
|
+
mon = Monitor(target)
|
|
191
|
+
await mon.start()
|
|
192
|
+
monitors.append(mon)
|
|
193
|
+
queues.append(mon.subscribe())
|
|
194
|
+
multi_task = asyncio.create_task(_forward_metrics_multi(websocket, queues, targets))
|
|
195
|
+
continue
|
|
57
196
|
|
|
58
197
|
if target := settings.targets_by_url.get(base_url):
|
|
59
198
|
LOGGER.info("Gathering metrics for: %s", target["name"])
|
|
@@ -69,14 +208,24 @@ async def websocket_endpoint(websocket: WebSocket) -> None:
|
|
|
69
208
|
q = monitor.subscribe()
|
|
70
209
|
|
|
71
210
|
# start forwarding metrics
|
|
72
|
-
asyncio.create_task(_forward_metrics(websocket, q))
|
|
211
|
+
forward_task = asyncio.create_task(_forward_metrics(websocket, q))
|
|
73
212
|
except WebSocketDisconnect:
|
|
74
213
|
pass
|
|
75
214
|
except Exception as err:
|
|
76
215
|
LOGGER.error("WS error: %s", err)
|
|
77
216
|
|
|
78
217
|
# cleanup
|
|
218
|
+
if forward_task:
|
|
219
|
+
forward_task.cancel()
|
|
220
|
+
|
|
79
221
|
if monitor:
|
|
80
222
|
if q:
|
|
81
223
|
monitor.unsubscribe(q)
|
|
82
224
|
await monitor.stop()
|
|
225
|
+
|
|
226
|
+
if multi_task:
|
|
227
|
+
multi_task.cancel()
|
|
228
|
+
|
|
229
|
+
for idx, mon in enumerate(monitors):
|
|
230
|
+
mon.unsubscribe(queues[idx])
|
|
231
|
+
await mon.stop()
|
pyobservability/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "1.
|
|
1
|
+
__version__ = "1.4.0"
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
pyobservability/__init__.py,sha256=yVBLyTohBiBKp0Otyl04IggPh8mhg3Er25u6eFyxMto,2618
|
|
2
|
+
pyobservability/main.py,sha256=EJ49ENDnKy07jKHQ0IYEVmsyc_4O3uxCHpn1faQD5VM,4534
|
|
3
|
+
pyobservability/monitor.py,sha256=i_Xf_DB-qLOp1b9wryekjwHIM8AnMrGTkuEg7e08bcM,7539
|
|
4
|
+
pyobservability/transport.py,sha256=S-84mgf-9yMj0H7VSAmueW9yosX_1XxdyNJC2EuQHQQ,8493
|
|
5
|
+
pyobservability/version.py,sha256=8UhoYEXHs1Oai7BW_ExBmuwWnRI-yMG_u1fQAXMizHQ,22
|
|
6
|
+
pyobservability/config/enums.py,sha256=EhvD9kB5EMW3ARxr5KmISmf-rP3D4IKqOIjw6Tb8SB8,294
|
|
7
|
+
pyobservability/config/settings.py,sha256=ylhiT0SARHuzkyon_1otgsO74AfA6aiUKp5uczZQj08,5980
|
|
8
|
+
pyobservability/static/app.js,sha256=lel2ZZH-ifjehzy2d-5UQxOhnBIHxla05xjokW1nXXA,33919
|
|
9
|
+
pyobservability/static/styles.css,sha256=0Vagj7nDac27JC0M870V3yqc1XN4rB5pDYuy4zjit3c,7618
|
|
10
|
+
pyobservability/templates/index.html,sha256=JdGn7Bg9w-4zzcMzX78p0q5760ao7sioaqB1s_8d0Fs,8421
|
|
11
|
+
pyobservability-1.4.0.dist-info/licenses/LICENSE,sha256=_sOIKJWdD2o1WwwDIwYB2qTP2nlSWqT5Tyg9jr1Xa4w,1070
|
|
12
|
+
pyobservability-1.4.0.dist-info/METADATA,sha256=DHbIRiPU0Vi_5PjsT0k5NgMInO8_YgY5azP9fGQM8VQ,6537
|
|
13
|
+
pyobservability-1.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
14
|
+
pyobservability-1.4.0.dist-info/entry_points.txt,sha256=DSGIr_VA8Tb3FYa2iNUYpf55eAvuFCAoInNS4ngXaME,57
|
|
15
|
+
pyobservability-1.4.0.dist-info/top_level.txt,sha256=p20T0EmihDYW1uMintRXr7X9bg3XWYKyoSbBHOVC1xI,16
|
|
16
|
+
pyobservability-1.4.0.dist-info/RECORD,,
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
pyobservability/__init__.py,sha256=yVBLyTohBiBKp0Otyl04IggPh8mhg3Er25u6eFyxMto,2618
|
|
2
|
-
pyobservability/main.py,sha256=6Ozyxi-XVq9ojkrTEjw9oYWJaW49JJZ-ezBNzEM951Y,4503
|
|
3
|
-
pyobservability/monitor.py,sha256=Y38zDJFPDbQqz_2jQcNmJBnRpeobC_SV2uhN-13e9RU,7591
|
|
4
|
-
pyobservability/transport.py,sha256=zHLAodX20bKkPOuacKjzv1Dqj3JbNAB75o1ABwzum0U,2534
|
|
5
|
-
pyobservability/version.py,sha256=F5mW07pSyGrqDNY2Ehr-UpDzpBtN-FsYU0QGZWf6PJE,22
|
|
6
|
-
pyobservability/config/enums.py,sha256=EhvD9kB5EMW3ARxr5KmISmf-rP3D4IKqOIjw6Tb8SB8,294
|
|
7
|
-
pyobservability/config/settings.py,sha256=l1xtD8teYS5ozXWm6JqpyyrcQsu95Syjs02xsd0m6MI,5886
|
|
8
|
-
pyobservability/static/app.js,sha256=ngeSsp8mCkF6XqZpXhwqv-EtNI8NmrMYxNgQtaKB4WE,26675
|
|
9
|
-
pyobservability/static/styles.css,sha256=4-VCDhzv_FnrvUffETTHkNnyGOFDUJU0qz2eLHCz_QI,5569
|
|
10
|
-
pyobservability/templates/index.html,sha256=OvFDydHMquBtFr_2cy3iXP0I-pax6_RKBJinVVEIzQY,7193
|
|
11
|
-
pyobservability-1.3.0.dist-info/licenses/LICENSE,sha256=_sOIKJWdD2o1WwwDIwYB2qTP2nlSWqT5Tyg9jr1Xa4w,1070
|
|
12
|
-
pyobservability-1.3.0.dist-info/METADATA,sha256=FJSF4L10qUjlQMYDKxJmuHrRkWmSLmOConJB5MKufOY,6537
|
|
13
|
-
pyobservability-1.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
14
|
-
pyobservability-1.3.0.dist-info/entry_points.txt,sha256=DSGIr_VA8Tb3FYa2iNUYpf55eAvuFCAoInNS4ngXaME,57
|
|
15
|
-
pyobservability-1.3.0.dist-info/top_level.txt,sha256=p20T0EmihDYW1uMintRXr7X9bg3XWYKyoSbBHOVC1xI,16
|
|
16
|
-
pyobservability-1.3.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|