PyObservability 1.3.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -51,6 +51,7 @@ def detailed_log_config(filename: str | None = None, debug: bool = False) -> Dic
51
51
  "uvicorn": {"handlers": ["default"], "level": level},
52
52
  "uvicorn.error": {"handlers": ["default"], "level": level, "propagate": False},
53
53
  "uvicorn.access": {"handlers": ["default"], "level": level, "propagate": False},
54
+ "uvicorn.default": {"handlers": ["default"], "level": level, "propagate": False},
54
55
  },
55
56
  "root": {"handlers": ["default"], "level": level},
56
57
  }
pyobservability/main.py CHANGED
@@ -2,7 +2,7 @@ import logging
2
2
  import pathlib
3
3
  import warnings
4
4
  from datetime import datetime
5
- from typing import Dict
5
+ from typing import Any, Dict
6
6
 
7
7
  import uiauth
8
8
  import uvicorn
@@ -39,7 +39,7 @@ async def index(request: Request):
39
39
  TemplateResponse:
40
40
  Rendered HTML template with targets and version.
41
41
  """
42
- args = dict(request=request, targets=settings.env.targets, version=__version__)
42
+ args: Dict[str, Any] = dict(request=request, targets=settings.env.targets, version=__version__)
43
43
  if settings.env.username and settings.env.password:
44
44
  args["logout"] = uiauth.enums.APIEndpoints.fastapi_logout.value
45
45
  return templates.TemplateResponse("index.html", args)
@@ -49,7 +49,7 @@ async def health() -> Dict[str, str]:
49
49
  """Health check endpoint.
50
50
 
51
51
  Returns:
52
- dict:
52
+ Dict[str, str]:
53
53
  Health status.
54
54
  """
55
55
  return {"status": "ok"}
@@ -174,7 +174,7 @@ class Monitor:
174
174
  "ts": asyncio.get_event_loop().time(),
175
175
  "data": [
176
176
  {
177
- "name": self.base_url,
177
+ "name": self.name,
178
178
  "base_url": self.base_url,
179
179
  "metrics": payload,
180
180
  }
@@ -188,13 +188,12 @@ class Monitor:
188
188
  _ = q.get_nowait()
189
189
  q.put_nowait(result)
190
190
  except Exception as err:
191
- LOGGER.debug("Stream error for %s: %s", self.base_url, err)
192
191
  if errors.get(self.base_url):
193
192
  if errors[self.base_url] < 10:
193
+ LOGGER.debug("Stream error for %s: %s", self.base_url, err)
194
194
  errors[self.base_url] += 1
195
195
  else:
196
- LOGGER.error(err.with_traceback())
197
- LOGGER.error("%s exceeded error threshold.", self.base_url)
196
+ LOGGER.error("Stream error for %s: %s", self.base_url, err)
198
197
 
199
198
  # notify subscribers before stopping
200
199
  error_msg = {
@@ -209,7 +208,7 @@ class Monitor:
209
208
  except asyncio.QueueFull as warn:
210
209
  LOGGER.warning(warn)
211
210
  _ = q.get_nowait()
212
- q.put_nowait(result)
211
+ q.put_nowait(error_msg)
213
212
  await self.stop()
214
213
  return
215
214
  else:
@@ -370,6 +370,149 @@
370
370
  const memChart = makeMainChart(memCtx, "Memory %");
371
371
  const loadChart = makeMainChart(loadCtx, "CPU Load");
372
372
 
373
+ // Unified metrics: three parallel charts (memory, CPU, disk)
374
+ const unifiedPanel = document.getElementById("unified-panel");
375
+ const unifiedLegend = document.getElementById("unified-legend");
376
+ const unifiedMemCtx = document.getElementById("unified-mem-chart").getContext("2d");
377
+ const unifiedCpuCtx = document.getElementById("unified-cpu-chart").getContext("2d");
378
+ const unifiedDiskCtx = document.getElementById("unified-disk-chart").getContext("2d");
379
+
380
+ let unifiedNodes = [];
381
+ const colorPalette = ["#63b3ff", "#ff99c8", "#7dd3fc", "#fbbf24", "#a3e635", "#f87171", "#c084fc", "#38bdf8"];
382
+ const nodeColor = {};
383
+ const unifiedCharts = {memory: null, cpu: null, disk: null};
384
+
385
+ function normalizeNodes(nodes) {
386
+ return nodes
387
+ .filter(node => node.base_url && node.base_url !== "*")
388
+ .sort((a, b) => (a.name || a.base_url).localeCompare(b.name || b.base_url));
389
+ }
390
+
391
+ function assignColors(nodes) {
392
+ nodes.forEach((node, idx) => {
393
+ nodeColor[node.base_url] = colorPalette[idx % colorPalette.length];
394
+ });
395
+ }
396
+
397
+ function renderLegend(nodes) {
398
+ unifiedLegend.innerHTML = "";
399
+ nodes.forEach(node => {
400
+ const item = document.createElement("div");
401
+ item.className = "unified-legend-item";
402
+ item.innerHTML = `<span class="legend-dot" style="background:${nodeColor[node.base_url]}"></span>${node.name || node.base_url}`;
403
+ unifiedLegend.appendChild(item);
404
+ });
405
+ }
406
+
407
+ function makeUnifiedChart(ctx, nodes) {
408
+ return new Chart(ctx, {
409
+ type: "line",
410
+ data: {
411
+ labels: Array(MAX_POINTS).fill(""),
412
+ datasets: nodes.map(node => ({
413
+ label: node.name || node.base_url,
414
+ meta: {base_url: node.base_url},
415
+ data: Array(MAX_POINTS).fill(null),
416
+ borderColor: nodeColor[node.base_url],
417
+ backgroundColor: `${nodeColor[node.base_url]}33`,
418
+ borderWidth: 2,
419
+ fill: false,
420
+ tension: 0.3,
421
+ pointRadius: 0,
422
+ }))
423
+ },
424
+ options: {
425
+ animation: false,
426
+ responsive: true,
427
+ maintainAspectRatio: false,
428
+ scales: {
429
+ x: {display: false},
430
+ y: {beginAtZero: true, suggestedMax: 100}
431
+ },
432
+ plugins: {legend: {display: false}}
433
+ }
434
+ });
435
+ }
436
+
437
+ function setUnifiedMode(enabled) {
438
+ document.body.classList.toggle("unified-mode", enabled);
439
+ if (!enabled) {
440
+ unifiedPanel.classList.add("hidden");
441
+ }
442
+ }
443
+
444
+ function ensureUnifiedChart(metrics) {
445
+ const nodes = normalizeNodes(metrics);
446
+ if (!nodes.length) return false;
447
+ const changed =
448
+ nodes.length !== unifiedNodes.length ||
449
+ nodes.some((node, idx) => unifiedNodes[idx]?.base_url !== node.base_url);
450
+
451
+ if (changed) {
452
+ // Destroy any existing unified charts
453
+ Object.keys(unifiedCharts).forEach(key => {
454
+ if (unifiedCharts[key]) {
455
+ unifiedCharts[key].destroy();
456
+ unifiedCharts[key] = null;
457
+ }
458
+ });
459
+ assignColors(nodes);
460
+ unifiedCharts.memory = makeUnifiedChart(unifiedMemCtx, nodes);
461
+ unifiedCharts.cpu = makeUnifiedChart(unifiedCpuCtx, nodes);
462
+ unifiedCharts.disk = makeUnifiedChart(unifiedDiskCtx, nodes);
463
+ unifiedNodes = nodes;
464
+ }
465
+
466
+ unifiedPanel.classList.remove("hidden");
467
+ renderLegend(nodes);
468
+ return true;
469
+ }
470
+
471
+ function sampleForMetric(host, metric) {
472
+ if (!host.metrics) return null;
473
+ if (metric === "memory") return host.metrics.memory_info?.percent ?? null;
474
+ if (metric === "cpu") {
475
+ const values = (host.metrics.cpu_usage || [])
476
+ .map(v => Number(v))
477
+ .filter(Number.isFinite);
478
+ if (!values.length) return null;
479
+ return values.reduce((a, b) => a + b, 0) / values.length;
480
+ }
481
+ if (metric === "disk") {
482
+ const m = host.metrics;
483
+ if (m.disk_info && m.disk_info[0]) {
484
+ const m = host.metrics;
485
+ const agg = aggregateDiskInfo(m.disk_info);
486
+ return agg.percent;
487
+ }
488
+ return host.metrics.disk_info?.[0]?.percent ?? null;
489
+ }
490
+ return null;
491
+ }
492
+
493
+ function updateUnified(metrics) {
494
+ const ts = new Date().toLocaleTimeString();
495
+ const metricKeys = ["memory", "cpu", "disk"];
496
+ metricKeys.forEach(metric => {
497
+ const chart = unifiedCharts[metric];
498
+ if (!chart) return;
499
+
500
+ chart.data.labels.push(ts);
501
+ if (chart.data.labels.length > MAX_POINTS) {
502
+ chart.data.labels.shift();
503
+ }
504
+
505
+ chart.data.datasets.forEach(ds => {
506
+ const host = metrics.find(h => h.base_url === ds.meta.base_url);
507
+ const value = host ? sampleForMetric(host, metric) : null;
508
+ ds.data.push(value);
509
+ if (ds.data.length > MAX_POINTS) ds.data.shift();
510
+ });
511
+
512
+ chart.update("none");
513
+ });
514
+ }
515
+
373
516
  // ------------------------------------------------------------
374
517
  // CORE CHARTS
375
518
  // ------------------------------------------------------------
@@ -398,9 +541,9 @@
398
541
  return coreMini[coreName] || createCoreChart(coreName);
399
542
  }
400
543
 
401
- function pruneOldCores(latest) {
544
+ function pruneOldCores(keep) {
402
545
  for (const name of Object.keys(coreMini)) {
403
- if (!latest.includes(name)) {
546
+ if (!keep.includes(name)) {
404
547
  try {
405
548
  coreMini[name].chart.destroy();
406
549
  } catch {
@@ -440,11 +583,23 @@
440
583
  resetChart(memChart);
441
584
  resetChart(loadChart);
442
585
 
586
+ // Reset unified charts as well
587
+ Object.keys(unifiedCharts).forEach(key => {
588
+ const chart = unifiedCharts[key];
589
+ if (chart) {
590
+ try {
591
+ chart.destroy();
592
+ } catch {}
593
+ unifiedCharts[key] = null;
594
+ }
595
+ });
596
+ unifiedNodes = [];
597
+ unifiedPanel.classList.add("hidden");
598
+
443
599
  for (const name of Object.keys(coreMini)) {
444
600
  try {
445
601
  coreMini[name].chart.destroy();
446
- } catch {
447
- }
602
+ } catch {}
448
603
  coreMini[name].el.remove();
449
604
  delete coreMini[name];
450
605
  }
@@ -460,6 +615,19 @@
460
615
  // ------------------------------------------------------------
461
616
  // MISC HELPERS
462
617
  // ------------------------------------------------------------
618
+ function aggregateDiskInfo(diskInfo) {
619
+ let totalDisk = 0;
620
+ let usedDisk = 0;
621
+ let freeDisk = 0;
622
+ diskInfo.forEach(d => {
623
+ totalDisk += num(d.total || 0);
624
+ usedDisk += num(d.used || 0);
625
+ freeDisk += num(d.free || 0);
626
+ });
627
+ let percentDisk = totalDisk > 0 ? (usedDisk / totalDisk) * 100 : 0.0;
628
+ return {"total": totalDisk, "used": usedDisk, "free": freeDisk, "percent": percentDisk};
629
+ }
630
+
463
631
  function pushPoint(chart, value) {
464
632
  const ts = new Date().toLocaleTimeString();
465
633
  chart.data.labels.push(ts);
@@ -519,6 +687,16 @@
519
687
 
520
688
  const now = new Date().toLocaleTimeString();
521
689
 
690
+ if (selectedBase === "*") {
691
+ if (ensureUnifiedChart(list)) {
692
+ setUnifiedMode(true);
693
+ updateUnified(list);
694
+ }
695
+ return;
696
+ }
697
+
698
+ setUnifiedMode(false);
699
+
522
700
  for (const host of list) {
523
701
  if (host.base_url !== selectedBase) continue;
524
702
  const m = host.metrics || {};
@@ -541,17 +719,19 @@
541
719
  `GPU: ${m.gpu_name || "-"}`;
542
720
 
543
721
  if (m.disk_info && m.disk_info[0]) {
544
- const d = m.disk_info[0];
722
+ const agg = aggregateDiskInfo(m.disk_info);
545
723
  diskEl.textContent =
546
- `Total: ${formatBytes(d.total)}\n` +
547
- `Used: ${formatBytes(d.used)}\n` +
548
- `Free: ${formatBytes(d.free)}`;
724
+ `Total: ${formatBytes(agg.total)}\n` +
725
+ `Used: ${formatBytes(agg.used)}\n` +
726
+ `Free: ${formatBytes(agg.free)}\n` +
727
+ `Percent: ${round2(agg.percent)}%`;
549
728
  }
550
729
 
551
730
  if (m.memory_info) {
552
731
  memEl.textContent =
553
732
  `Total: ${formatBytes(m.memory_info.total)}\n` +
554
733
  `Used: ${formatBytes(m.memory_info.used)}\n` +
734
+ `Free: ${formatBytes(m.memory_info.free)}\n` +
555
735
  `Percent: ${round2(m.memory_info.percent)}%`;
556
736
  pushPoint(memChart, num(m.memory_info.percent));
557
737
  }
@@ -644,11 +824,30 @@
644
824
  hideSpinner("certificates-table");
645
825
  }
646
826
  }
827
+
828
+ if (selectedBase === "*") {
829
+ if (ensureUnifiedChart(list)) {
830
+ updateUnified(list);
831
+ }
832
+ } else {
833
+ unifiedPanel.classList.add("hidden");
834
+ unifiedNodes = [];
835
+ Object.keys(unifiedCharts).forEach(key => {
836
+ if (unifiedCharts[key]) {
837
+ unifiedCharts[key].destroy();
838
+ unifiedCharts[key] = null;
839
+ }
840
+ });
841
+ }
647
842
  }
648
843
 
649
844
  // ------------------------------------------------------------
650
845
  // EVENT BINDINGS
651
846
  // ------------------------------------------------------------
847
+ targets.push({
848
+ base_url: "*",
849
+ name: "*"
850
+ });
652
851
  targets.forEach(t => {
653
852
  const opt = document.createElement("option");
654
853
  opt.value = t.base_url;
@@ -664,6 +863,7 @@
664
863
  resetUI();
665
864
  resetTables();
666
865
  showAllSpinners();
866
+ if (selectedBase !== "*") unifiedPanel.classList.add("hidden");
667
867
  ws.send(JSON.stringify({type: "select_target", base_url: selectedBase}));
668
868
  });
669
869
 
@@ -265,6 +265,10 @@ input#proc-filter {
265
265
  .meta-row {
266
266
  grid-template-columns: 1fr;
267
267
  }
268
+
269
+ .unified-grid {
270
+ grid-template-columns: 1fr;
271
+ }
268
272
  }
269
273
 
270
274
  /* ---------------- SPINNER ------------------- */
@@ -324,3 +328,111 @@ input#proc-filter {
324
328
  margin-bottom: 4px;
325
329
  opacity: 0.85;
326
330
  }
331
+
332
+ .panel-helper {
333
+ margin: 0 0 8px;
334
+ font-size: 12px;
335
+ color: var(--muted);
336
+ }
337
+
338
+ .metric-switch {
339
+ display: inline-flex;
340
+ border: 1px solid rgba(255, 255, 255, 0.08);
341
+ border-radius: 4px;
342
+ overflow: hidden;
343
+ }
344
+
345
+ .metric-btn {
346
+ background: transparent;
347
+ color: inherit;
348
+ border: none;
349
+ padding: 6px 12px;
350
+ font-size: 12px;
351
+ cursor: pointer;
352
+ transition: background 0.2s, color 0.2s;
353
+ }
354
+
355
+ .metric-btn:not(.active):hover {
356
+ background: rgba(255, 255, 255, 0.05);
357
+ }
358
+
359
+ .metric-btn.active {
360
+ background: var(--accent);
361
+ color: #020617;
362
+ }
363
+
364
+ .unified-header {
365
+ align-items: center;
366
+ justify-content: center;
367
+ gap: 12px;
368
+ }
369
+
370
+ .unified-title-group {
371
+ display: flex;
372
+ flex-direction: column;
373
+ align-items: center;
374
+ justify-content: center;
375
+ text-align: center;
376
+ gap: 2px;
377
+ }
378
+
379
+ .unified-indicator {
380
+ font-size: 12px;
381
+ text-transform: uppercase;
382
+ letter-spacing: 0.05em;
383
+ color: var(--muted);
384
+ }
385
+
386
+ .unified-legend {
387
+ display: flex;
388
+ flex-wrap: wrap;
389
+ gap: 8px;
390
+ margin-top: 10px;
391
+ }
392
+
393
+ .unified-legend-item {
394
+ display: flex;
395
+ align-items: center;
396
+ gap: 6px;
397
+ padding: 4px 8px;
398
+ border-radius: 4px;
399
+ background: rgba(255, 255, 255, 0.05);
400
+ font-size: 12px;
401
+ }
402
+
403
+ .legend-dot {
404
+ width: 10px;
405
+ height: 10px;
406
+ border-radius: 50%;
407
+ }
408
+
409
+ /* New unified-grid layout for three unified charts */
410
+ .unified-grid {
411
+ display: grid;
412
+ grid-template-columns: repeat(3, minmax(0, 1fr));
413
+ gap: 12px;
414
+ margin-top: 8px;
415
+ }
416
+
417
+ .unified-metric-panel {
418
+ background: var(--panel);
419
+ border-radius: 8px;
420
+ padding: 8px 8px 4px;
421
+ box-shadow: 0 4px 12px rgba(2, 6, 23, 0.6);
422
+ }
423
+
424
+ .unified-metric-panel .panel-header h3 {
425
+ font-size: 13px;
426
+ }
427
+
428
+ body.unified-mode .meta-row,
429
+ body.unified-mode .charts-row > :not(#unified-panel),
430
+ body.unified-mode .details-row,
431
+ body.unified-mode .tables-row,
432
+ body.unified-mode footer {
433
+ display: none !important;
434
+ }
435
+
436
+ body.unified-mode #unified-panel {
437
+ flex: 1;
438
+ }
@@ -75,6 +75,31 @@
75
75
  <div class="panel-header"><h3>Memory (%)</h3></div>
76
76
  <canvas id="mem-chart" class="chart"></canvas>
77
77
  </div>
78
+
79
+ <div class="panel hidden" id="unified-panel">
80
+ <div class="panel-header unified-header">
81
+ <div class="unified-title-group">
82
+ <h3>Aggregated Metrics</h3>
83
+ <span class="unified-indicator">Holistic node comparison</span>
84
+ </div>
85
+ </div>
86
+ {# <p class="panel-helper">Select "All nodes" (*) to compare every host.</p> #}
87
+ <div class="unified-grid">
88
+ <div class="unified-metric-panel">
89
+ <div class="panel-header"><h3>Memory (%)</h3></div>
90
+ <canvas id="unified-mem-chart" class="chart"></canvas>
91
+ </div>
92
+ <div class="unified-metric-panel">
93
+ <div class="panel-header"><h3>CPU Avg (%)</h3></div>
94
+ <canvas id="unified-cpu-chart" class="chart"></canvas>
95
+ </div>
96
+ <div class="unified-metric-panel">
97
+ <div class="panel-header"><h3>Disk (%)</h3></div>
98
+ <canvas id="unified-disk-chart" class="chart"></canvas>
99
+ </div>
100
+ </div>
101
+ <div id="unified-legend" class="unified-legend"></div>
102
+ </div>
78
103
  </section>
79
104
 
80
105
  <section class="details-row">
@@ -1,6 +1,7 @@
1
1
  import asyncio
2
2
  import json
3
3
  import logging
4
+ from typing import Dict, List
4
5
 
5
6
  from fastapi import WebSocket, WebSocketDisconnect
6
7
 
@@ -22,6 +23,99 @@ async def _forward_metrics(websocket: WebSocket, q: asyncio.Queue) -> None:
22
23
  await websocket.send_json(payload)
23
24
 
24
25
 
26
+ def _normalize_targets() -> List[Dict[str, str]]:
27
+ """Return configuration targets sorted so legend colors are consistent."""
28
+ return sorted(settings.env.targets, key=lambda t: t["name"].lower())
29
+
30
+
31
+ async def _forward_metrics_multi(
32
+ websocket: WebSocket,
33
+ queues: List[asyncio.Queue],
34
+ targets: List[Dict[str, str]],
35
+ ) -> None:
36
+ """Fan-in metrics from multiple monitors and emit once every node updates.
37
+
38
+ Args:
39
+ websocket: FastAPI WebSocket connection.
40
+ queues: List of asyncio.Queues from multiple monitors.
41
+ targets: List of target configurations corresponding to the queues.
42
+
43
+ Notes:
44
+ This is resilient to individual node failures.
45
+ If a monitor starts emitting an ``error`` payload (e.g. exceeded error threshold),
46
+ that node is marked as failed and removed from the unified stream,
47
+ so that the remaining nodes keep updating in the UI.
48
+ """
49
+ # Track active (non-failed) indices
50
+ active = [True] * len(queues)
51
+
52
+ # Latest payload per node keyed by base_url
53
+ latest: Dict[str, dict] = {}
54
+ try:
55
+ while True:
56
+ # If all nodes have failed, stop the unified stream loop.
57
+ if not any(active):
58
+ LOGGER.info("All targets in unified stream have failed; stopping multi-node forwarder")
59
+ return
60
+
61
+ # Wait for at least one payload from any active queue.
62
+ # We gather one payload per active queue in a "round", but we do
63
+ # not block forever on failed queues because we mark them inactive
64
+ # as soon as they send an error.
65
+ for idx, q in enumerate(queues):
66
+ if not active[idx]:
67
+ continue
68
+
69
+ payload = await q.get()
70
+
71
+ # Handle error payloads from monitors: mark this node as failed
72
+ # and notify the UI once, then skip it from future
73
+ # rounds so the rest of the nodes continue streaming.
74
+ if isinstance(payload, dict) and payload.get("type") == "error":
75
+ base_url = targets[idx]["base_url"]
76
+ LOGGER.warning("Unified stream: target %s reported error and will be skipped", base_url)
77
+
78
+ # Forward the error to the websocket so the UI can show it.
79
+ try:
80
+ await websocket.send_json(payload)
81
+ except Exception as send_err: # pragma: no cover - defensive
82
+ LOGGER.debug("Failed to send error payload to WS: %s", send_err)
83
+
84
+ # Mark this target as inactive and remove its latest sample
85
+ active[idx] = False
86
+ latest.pop(base_url, None)
87
+ continue
88
+
89
+ # Normal metrics payload: remember latest per base_url
90
+ base_url = targets[idx]["base_url"]
91
+ latest[base_url] = payload
92
+
93
+ # Build merged message from all currently active targets that have
94
+ # produced at least one metrics payload.
95
+ merged = {
96
+ "type": "metrics",
97
+ "ts": asyncio.get_event_loop().time(),
98
+ "data": [],
99
+ }
100
+
101
+ for idx, target in enumerate(targets):
102
+ if not active[idx]:
103
+ continue
104
+ sample = latest.get(target["base_url"])
105
+ if not sample:
106
+ continue
107
+ merged["data"].extend(sample.get("data", []))
108
+
109
+ # If we have no data (e.g. new round before any active node
110
+ # produced metrics), skip sending to avoid spamming empty payloads.
111
+ if not merged["data"]:
112
+ continue
113
+
114
+ await websocket.send_json(merged)
115
+ except asyncio.CancelledError:
116
+ LOGGER.debug("Unified stream task cancelled")
117
+
118
+
25
119
  async def websocket_endpoint(websocket: WebSocket) -> None:
26
120
  """Websocket endpoint to handle observability data streaming.
27
121
 
@@ -32,6 +126,10 @@ async def websocket_endpoint(websocket: WebSocket) -> None:
32
126
 
33
127
  monitor: Monitor | None = None
34
128
  q: asyncio.Queue | None = None
129
+ monitors: List[Monitor] = []
130
+ queues: List[asyncio.Queue] = []
131
+ multi_task: asyncio.Task | None = None
132
+ forward_task: asyncio.Task | None = None
35
133
 
36
134
  try:
37
135
  while True:
@@ -42,6 +140,15 @@ async def websocket_endpoint(websocket: WebSocket) -> None:
42
140
  await monitor.update_flags(
43
141
  all_services=data.get("all_services", False),
44
142
  )
143
+ elif monitors:
144
+ await asyncio.gather(
145
+ *(
146
+ mon.update_flags(
147
+ all_services=data.get("all_services", False),
148
+ )
149
+ for mon in monitors
150
+ )
151
+ )
45
152
  continue
46
153
 
47
154
  # -------------------------------------------
@@ -50,10 +157,42 @@ async def websocket_endpoint(websocket: WebSocket) -> None:
50
157
  if data.get("type") == "select_target":
51
158
  base_url = data["base_url"]
52
159
 
53
- # stop old monitor
160
+ # stop old monitor / tasks
161
+ if forward_task:
162
+ LOGGER.info("Stopping previous forwarder task")
163
+ forward_task.cancel()
164
+ forward_task = None
165
+
54
166
  if monitor:
167
+ LOGGER.info("Stopping previous monitor task")
55
168
  monitor.unsubscribe(q)
56
169
  await monitor.stop()
170
+ monitor = None
171
+ q = None
172
+
173
+ if multi_task:
174
+ LOGGER.info("Stopping previous multi-target forwarder task")
175
+ multi_task.cancel()
176
+ multi_task = None
177
+
178
+ LOGGER.info("Unsubscribing from previous monitors") if monitor else None
179
+ for idx, mon in enumerate(monitors):
180
+ mon.unsubscribe(queues[idx])
181
+ await mon.stop()
182
+
183
+ monitors.clear()
184
+ queues.clear()
185
+
186
+ if base_url == "*":
187
+ LOGGER.info("Gathering metrics for all targets in unified stream")
188
+ targets = _normalize_targets()
189
+ for target in targets:
190
+ mon = Monitor(target)
191
+ await mon.start()
192
+ monitors.append(mon)
193
+ queues.append(mon.subscribe())
194
+ multi_task = asyncio.create_task(_forward_metrics_multi(websocket, queues, targets))
195
+ continue
57
196
 
58
197
  if target := settings.targets_by_url.get(base_url):
59
198
  LOGGER.info("Gathering metrics for: %s", target["name"])
@@ -69,14 +208,24 @@ async def websocket_endpoint(websocket: WebSocket) -> None:
69
208
  q = monitor.subscribe()
70
209
 
71
210
  # start forwarding metrics
72
- asyncio.create_task(_forward_metrics(websocket, q))
211
+ forward_task = asyncio.create_task(_forward_metrics(websocket, q))
73
212
  except WebSocketDisconnect:
74
213
  pass
75
214
  except Exception as err:
76
215
  LOGGER.error("WS error: %s", err)
77
216
 
78
217
  # cleanup
218
+ if forward_task:
219
+ forward_task.cancel()
220
+
79
221
  if monitor:
80
222
  if q:
81
223
  monitor.unsubscribe(q)
82
224
  await monitor.stop()
225
+
226
+ if multi_task:
227
+ multi_task.cancel()
228
+
229
+ for idx, mon in enumerate(monitors):
230
+ mon.unsubscribe(queues[idx])
231
+ await mon.stop()
@@ -1 +1 @@
1
- __version__ = "1.3.0"
1
+ __version__ = "1.4.0"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: PyObservability
3
- Version: 1.3.0
3
+ Version: 1.4.0
4
4
  Summary: Lightweight OS-agnostic observability UI for PyNinja
5
5
  Author-email: Vignesh Rao <svignesh1793@gmail.com>
6
6
  License: MIT License
@@ -0,0 +1,16 @@
1
+ pyobservability/__init__.py,sha256=yVBLyTohBiBKp0Otyl04IggPh8mhg3Er25u6eFyxMto,2618
2
+ pyobservability/main.py,sha256=EJ49ENDnKy07jKHQ0IYEVmsyc_4O3uxCHpn1faQD5VM,4534
3
+ pyobservability/monitor.py,sha256=i_Xf_DB-qLOp1b9wryekjwHIM8AnMrGTkuEg7e08bcM,7539
4
+ pyobservability/transport.py,sha256=S-84mgf-9yMj0H7VSAmueW9yosX_1XxdyNJC2EuQHQQ,8493
5
+ pyobservability/version.py,sha256=8UhoYEXHs1Oai7BW_ExBmuwWnRI-yMG_u1fQAXMizHQ,22
6
+ pyobservability/config/enums.py,sha256=EhvD9kB5EMW3ARxr5KmISmf-rP3D4IKqOIjw6Tb8SB8,294
7
+ pyobservability/config/settings.py,sha256=ylhiT0SARHuzkyon_1otgsO74AfA6aiUKp5uczZQj08,5980
8
+ pyobservability/static/app.js,sha256=lel2ZZH-ifjehzy2d-5UQxOhnBIHxla05xjokW1nXXA,33919
9
+ pyobservability/static/styles.css,sha256=0Vagj7nDac27JC0M870V3yqc1XN4rB5pDYuy4zjit3c,7618
10
+ pyobservability/templates/index.html,sha256=JdGn7Bg9w-4zzcMzX78p0q5760ao7sioaqB1s_8d0Fs,8421
11
+ pyobservability-1.4.0.dist-info/licenses/LICENSE,sha256=_sOIKJWdD2o1WwwDIwYB2qTP2nlSWqT5Tyg9jr1Xa4w,1070
12
+ pyobservability-1.4.0.dist-info/METADATA,sha256=DHbIRiPU0Vi_5PjsT0k5NgMInO8_YgY5azP9fGQM8VQ,6537
13
+ pyobservability-1.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
14
+ pyobservability-1.4.0.dist-info/entry_points.txt,sha256=DSGIr_VA8Tb3FYa2iNUYpf55eAvuFCAoInNS4ngXaME,57
15
+ pyobservability-1.4.0.dist-info/top_level.txt,sha256=p20T0EmihDYW1uMintRXr7X9bg3XWYKyoSbBHOVC1xI,16
16
+ pyobservability-1.4.0.dist-info/RECORD,,
@@ -1,16 +0,0 @@
1
- pyobservability/__init__.py,sha256=yVBLyTohBiBKp0Otyl04IggPh8mhg3Er25u6eFyxMto,2618
2
- pyobservability/main.py,sha256=6Ozyxi-XVq9ojkrTEjw9oYWJaW49JJZ-ezBNzEM951Y,4503
3
- pyobservability/monitor.py,sha256=Y38zDJFPDbQqz_2jQcNmJBnRpeobC_SV2uhN-13e9RU,7591
4
- pyobservability/transport.py,sha256=zHLAodX20bKkPOuacKjzv1Dqj3JbNAB75o1ABwzum0U,2534
5
- pyobservability/version.py,sha256=F5mW07pSyGrqDNY2Ehr-UpDzpBtN-FsYU0QGZWf6PJE,22
6
- pyobservability/config/enums.py,sha256=EhvD9kB5EMW3ARxr5KmISmf-rP3D4IKqOIjw6Tb8SB8,294
7
- pyobservability/config/settings.py,sha256=l1xtD8teYS5ozXWm6JqpyyrcQsu95Syjs02xsd0m6MI,5886
8
- pyobservability/static/app.js,sha256=ngeSsp8mCkF6XqZpXhwqv-EtNI8NmrMYxNgQtaKB4WE,26675
9
- pyobservability/static/styles.css,sha256=4-VCDhzv_FnrvUffETTHkNnyGOFDUJU0qz2eLHCz_QI,5569
10
- pyobservability/templates/index.html,sha256=OvFDydHMquBtFr_2cy3iXP0I-pax6_RKBJinVVEIzQY,7193
11
- pyobservability-1.3.0.dist-info/licenses/LICENSE,sha256=_sOIKJWdD2o1WwwDIwYB2qTP2nlSWqT5Tyg9jr1Xa4w,1070
12
- pyobservability-1.3.0.dist-info/METADATA,sha256=FJSF4L10qUjlQMYDKxJmuHrRkWmSLmOConJB5MKufOY,6537
13
- pyobservability-1.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
14
- pyobservability-1.3.0.dist-info/entry_points.txt,sha256=DSGIr_VA8Tb3FYa2iNUYpf55eAvuFCAoInNS4ngXaME,57
15
- pyobservability-1.3.0.dist-info/top_level.txt,sha256=p20T0EmihDYW1uMintRXr7X9bg3XWYKyoSbBHOVC1xI,16
16
- pyobservability-1.3.0.dist-info/RECORD,,