serverish 1.5.0__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {serverish-1.5.0 → serverish-2.0.0}/PKG-INFO +10 -4
- {serverish-1.5.0 → serverish-2.0.0}/README.md +5 -1
- {serverish-1.5.0 → serverish-2.0.0}/pyproject.toml +5 -3
- {serverish-1.5.0 → serverish-2.0.0}/serverish/base/__init__.py +1 -1
- {serverish-1.5.0 → serverish-2.0.0}/serverish/base/exceptions.py +1 -1
- {serverish-1.5.0 → serverish-2.0.0}/serverish/base/hasstatuses.py +25 -23
- serverish-2.0.0/serverish/base/status.py +225 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/base/task_manager.py +4 -4
- {serverish-1.5.0 → serverish-2.0.0}/serverish/connection/connection.py +13 -13
- {serverish-1.5.0 → serverish-2.0.0}/serverish/connection/connection_jets.py +19 -19
- {serverish-1.5.0 → serverish-2.0.0}/serverish/connection/connection_nats.py +74 -21
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/__init__.py +5 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/live_document.py +39 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/messenger.py +71 -0
- serverish-2.0.0/serverish/messenger/msg_cmd_pub.py +58 -0
- serverish-2.0.0/serverish/messenger/msg_cmd_sub.py +93 -0
- serverish-2.0.0/serverish/messenger/msg_core_pub.py +93 -0
- serverish-2.0.0/serverish/messenger/msg_core_read.py +164 -0
- serverish-2.0.0/serverish/messenger/msg_core_sub.py +116 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/msg_journal_pub.py +13 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/msg_progress_pub.py +15 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/msg_publisher.py +59 -5
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/msg_reader.py +146 -22
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/msg_rpc_req.py +1 -1
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/msg_rpc_resp.py +79 -0
- serverish-2.0.0/serverish/monitoring/__init__.py +53 -0
- serverish-2.0.0/serverish/monitoring/bridge.py +111 -0
- serverish-2.0.0/serverish/monitoring/factory.py +70 -0
- serverish-2.0.0/serverish/monitoring/monitored.py +411 -0
- serverish-2.0.0/serverish/monitoring/monitored_nats.py +126 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/schema/meta.schema.json +1 -1
- serverish-1.5.0/serverish/base/status.py +0 -132
- {serverish-1.5.0 → serverish-2.0.0}/LICENSE +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/__init__.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/base/asyncio_util_functions.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/base/collector.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/base/datetime.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/base/fifoset.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/base/idmanger.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/base/iterators.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/base/manageable.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/base/singleton.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/connection/__init__.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/logging.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/msg_callback_sub.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/msg_document_read.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/msg_journal_read.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/msg_progress_read.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/msg_single_pub.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/msg_single_read.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/messenger/msgvalidator.py +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/schema/config.schema.json +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/schema/default.schema.json +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/schema/include/timestamp.schema.json +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/schema/journal.schema.json +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/schema/program_current_state.schema.json +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/schema/program_state.schema.json +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/schema/progress.schema.json +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/schema/telemetry.schema.json +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/services/TODO.txt +0 -0
- {serverish-1.5.0 → serverish-2.0.0}/serverish/slog.py +0 -0
|
@@ -1,14 +1,16 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: serverish
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: helpers for server alike projects
|
|
5
|
-
|
|
5
|
+
License-File: LICENSE
|
|
6
6
|
Author: Mikołaj Kałuszyński
|
|
7
7
|
Requires-Python: >=3.10,<4.0
|
|
8
8
|
Classifier: Programming Language :: Python :: 3
|
|
9
9
|
Classifier: Programming Language :: Python :: 3.10
|
|
10
10
|
Classifier: Programming Language :: Python :: 3.11
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
12
14
|
Provides-Extra: dns
|
|
13
15
|
Provides-Extra: messenger
|
|
14
16
|
Requires-Dist: aiodns (>=3.2.0,<4.0.0) ; extra == "dns"
|
|
@@ -45,9 +47,13 @@ this will install `nats-py` package.
|
|
|
45
47
|
## Plans
|
|
46
48
|
- Decorators `@messenger.sub(subject)` (Web framework style)
|
|
47
49
|
- Progress with automatic increment for known durations e.g. `pro_pub.update(completed=1, when=+20)`
|
|
48
|
-
-
|
|
50
|
+
- Custom JetStream API replacing nats-py private internals access
|
|
49
51
|
|
|
50
52
|
## Changes
|
|
53
|
+
* 1.7 Subscription reliability: reader auto-recovers from NATS disconnects and consumer expiry,
|
|
54
|
+
RPC responder auto-resubscribes, `health_status` property on all drivers (reader, publisher,
|
|
55
|
+
RPC responder, connection) with reconnect counts, error tracking, slow consumer detection.
|
|
56
|
+
Comprehensive test suite (194 tests) with testcontainers-based NATS fixtures.
|
|
51
57
|
* 1.5 Introduces live documents (`LiveDocument`, `get_documentreader()`, `get_live_document()`) for auto-updating configuration from NATS.
|
|
52
58
|
* 1.4 Contains improvements to the network problems recovery and connection tracking.
|
|
53
59
|
* 1.1 Switches to `param` 2.* nad `py-nats` 1.7.*. Also, publisher may reraise exceptions or ignore them.
|
|
@@ -25,9 +25,13 @@ this will install `nats-py` package.
|
|
|
25
25
|
## Plans
|
|
26
26
|
- Decorators `@messenger.sub(subject)` (Web framework style)
|
|
27
27
|
- Progress with automatic increment for known durations e.g. `pro_pub.update(completed=1, when=+20)`
|
|
28
|
-
-
|
|
28
|
+
- Custom JetStream API replacing nats-py private internals access
|
|
29
29
|
|
|
30
30
|
## Changes
|
|
31
|
+
* 1.7 Subscription reliability: reader auto-recovers from NATS disconnects and consumer expiry,
|
|
32
|
+
RPC responder auto-resubscribes, `health_status` property on all drivers (reader, publisher,
|
|
33
|
+
RPC responder, connection) with reconnect counts, error tracking, slow consumer detection.
|
|
34
|
+
Comprehensive test suite (194 tests) with testcontainers-based NATS fixtures.
|
|
31
35
|
* 1.5 Introduces live documents (`LiveDocument`, `get_documentreader()`, `get_live_document()`) for auto-updating configuration from NATS.
|
|
32
36
|
* 1.4 Contains improvements to the network problems recovery and connection tracking.
|
|
33
37
|
* 1.1 Switches to `param` 2.* nad `py-nats` 1.7.*. Also, publisher may reraise exceptions or ignore them.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "serverish"
|
|
3
|
-
version = "
|
|
3
|
+
version = "2.0.0"
|
|
4
4
|
description = "helpers for server alike projects"
|
|
5
5
|
authors = ["Mikołaj Kałuszyński", "MMME team"]
|
|
6
6
|
readme = "README.md"
|
|
@@ -20,12 +20,14 @@ dns = ["aiodns"]
|
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
[tool.poetry.group.dev.dependencies]
|
|
23
|
-
pytest = "
|
|
24
|
-
pytest-asyncio = "
|
|
23
|
+
pytest = ">=8.2,<9"
|
|
24
|
+
pytest-asyncio = ">=0.24,<1"
|
|
25
25
|
nats-py = "^2.6.0"
|
|
26
26
|
jupyter = "*"
|
|
27
27
|
rich = "^13.5.2"
|
|
28
28
|
aiodns = "^3.1.1"
|
|
29
|
+
testcontainers = {extras = ["nats"], version = "^4.14.2"}
|
|
30
|
+
pytest-timeout = "^2.4.0"
|
|
29
31
|
|
|
30
32
|
|
|
31
33
|
[build-system]
|
|
@@ -18,7 +18,7 @@ class MessengerRequestNoResponders(MessengerRequestNoResponse):
|
|
|
18
18
|
|
|
19
19
|
class MessengerRequestJetStreamSubject(MessengerRequestNoResponse):
|
|
20
20
|
def __init__(self, subject:str) -> None:
|
|
21
|
-
super().__init__(f'Subject {subject} probably declared in
|
|
21
|
+
super().__init__(f'Subject {subject} probably declared in JetStream stream. '
|
|
22
22
|
f'Use pure NATS core subjects for RPC')
|
|
23
23
|
|
|
24
24
|
|
|
@@ -7,11 +7,11 @@ import param
|
|
|
7
7
|
|
|
8
8
|
from serverish.base.collector import Collector
|
|
9
9
|
from serverish.base.manageable import Manageable
|
|
10
|
-
from serverish.base.status import Status,
|
|
10
|
+
from serverish.base.status import Status, StatusReport
|
|
11
11
|
|
|
12
12
|
logger = logging.getLogger(__name__.rsplit('.')[-1])
|
|
13
13
|
|
|
14
|
-
CheckMethodType = Callable[[], Union[Awaitable[
|
|
14
|
+
CheckMethodType = Callable[[], Union[Awaitable[StatusReport], StatusReport]]
|
|
15
15
|
|
|
16
16
|
class HasStatuses(Manageable):
|
|
17
17
|
"""Controls the resource
|
|
@@ -23,9 +23,9 @@ class HasStatuses(Manageable):
|
|
|
23
23
|
|
|
24
24
|
"""
|
|
25
25
|
@staticmethod
|
|
26
|
-
def diagnose_dummy_ok() ->
|
|
27
|
-
"""Dummy diagnosis, returns
|
|
28
|
-
return
|
|
26
|
+
def diagnose_dummy_ok() -> StatusReport:
|
|
27
|
+
"""Dummy diagnosis, returns OK status."""
|
|
28
|
+
return StatusReport.ok(msg='Dummy OK')
|
|
29
29
|
|
|
30
30
|
status = param.Dict(default={}, doc='A dictionary to hold status information')
|
|
31
31
|
|
|
@@ -68,7 +68,7 @@ class HasStatuses(Manageable):
|
|
|
68
68
|
else:
|
|
69
69
|
self.check_methods = {**self.check_methods, **methods}
|
|
70
70
|
|
|
71
|
-
def set_status(self, key: str, value:
|
|
71
|
+
def set_status(self, key: str, value: StatusReport | None) -> None:
|
|
72
72
|
"""Sets status of the resource
|
|
73
73
|
|
|
74
74
|
A resource can have multiple statuses, e.g. 'main', 'dns', 'http', etc.
|
|
@@ -83,16 +83,16 @@ class HasStatuses(Manageable):
|
|
|
83
83
|
else:
|
|
84
84
|
self.status[key] = value
|
|
85
85
|
|
|
86
|
-
async def diagnose(self, no_deduce: bool = False) -> dict[str,
|
|
86
|
+
async def diagnose(self, no_deduce: bool = False) -> dict[str, StatusReport]:
|
|
87
87
|
"""Diagnoses the object
|
|
88
88
|
|
|
89
89
|
Diagnose by calling check methods and setting statuses.
|
|
90
90
|
Methods are called in order, values of later methods may be deduced from earlier ones:
|
|
91
|
-
if some method returns
|
|
92
|
-
is the same.
|
|
91
|
+
if some method returns a StatusReport with deduce_other=True, we deduce that the result
|
|
92
|
+
of all later methods is the same.
|
|
93
93
|
E.g. if 'ping' is OK, we deduce that 'dns' is OK too.
|
|
94
94
|
|
|
95
|
-
One can force all
|
|
95
|
+
One can force all diagnostics to be run by setting no_deduce=True.
|
|
96
96
|
|
|
97
97
|
Args:
|
|
98
98
|
no_deduce (bool, optional): If True, don't deduce statuses from other statuses. Defaults to False.
|
|
@@ -109,11 +109,11 @@ class HasStatuses(Manageable):
|
|
|
109
109
|
res = await asyncio.gather(*checking_tasks)
|
|
110
110
|
else:
|
|
111
111
|
res = []
|
|
112
|
-
deduced_status:
|
|
112
|
+
deduced_status: StatusReport | None = None
|
|
113
113
|
deduced_name: str | None = None
|
|
114
114
|
for n, m in self.check_methods.items():
|
|
115
115
|
if deduced_status is not None:
|
|
116
|
-
r =
|
|
116
|
+
r = StatusReport.deduced(deduced_status, f'Deduced from {deduced_name}')
|
|
117
117
|
else:
|
|
118
118
|
r = await calc_state(m)
|
|
119
119
|
if r.deduce_other:
|
|
@@ -122,16 +122,16 @@ class HasStatuses(Manageable):
|
|
|
122
122
|
res.append(r)
|
|
123
123
|
return {n: r for n, r in zip(self.check_methods.keys(), res)}
|
|
124
124
|
|
|
125
|
-
def diagnose_sync(self, no_deduce: bool = False) -> dict[str,
|
|
125
|
+
def diagnose_sync(self, no_deduce: bool = False) -> dict[str, StatusReport]:
|
|
126
126
|
"""Diagnoses the object skipping async methods
|
|
127
127
|
|
|
128
128
|
Diagnose by calling check methods and setting statuses.
|
|
129
129
|
Methods are called in order, values of later methods may be deduced from earlier ones:
|
|
130
|
-
if some method returns
|
|
131
|
-
is the same.
|
|
130
|
+
if some method returns a StatusReport with deduce_other=True, we deduce that the result
|
|
131
|
+
of all later methods is the same.
|
|
132
132
|
E.g. if 'ping' is OK, we deduce that 'dns' is OK too.
|
|
133
133
|
|
|
134
|
-
One can force all
|
|
134
|
+
One can force all diagnostics to be run by setting no_deduce=True.
|
|
135
135
|
|
|
136
136
|
Args:
|
|
137
137
|
no_deduce (bool, optional): If True, don't deduce statuses from other statuses. Defaults to False.
|
|
@@ -139,7 +139,7 @@ class HasStatuses(Manageable):
|
|
|
139
139
|
|
|
140
140
|
def calc_state(m):
|
|
141
141
|
if asyncio.iscoroutinefunction(m):
|
|
142
|
-
return
|
|
142
|
+
return StatusReport.unknown(msg='Skipped async test')
|
|
143
143
|
else:
|
|
144
144
|
return m()
|
|
145
145
|
|
|
@@ -147,11 +147,11 @@ class HasStatuses(Manageable):
|
|
|
147
147
|
res = [calc_state(m) for m in self.check_methods.values()]
|
|
148
148
|
else:
|
|
149
149
|
res = []
|
|
150
|
-
deduced_status:
|
|
150
|
+
deduced_status: StatusReport | None = None
|
|
151
151
|
deduced_name: str | None = None
|
|
152
152
|
for n, m in self.check_methods.items():
|
|
153
153
|
if deduced_status is not None:
|
|
154
|
-
r =
|
|
154
|
+
r = StatusReport.deduced(deduced_status, f'Deduced from {deduced_name}')
|
|
155
155
|
else:
|
|
156
156
|
r = calc_state(m)
|
|
157
157
|
if r.deduce_other:
|
|
@@ -184,15 +184,17 @@ class HasStatuses(Manageable):
|
|
|
184
184
|
for n, s in statuses.items():
|
|
185
185
|
self.set_status(n, s)
|
|
186
186
|
|
|
187
|
-
def point_of_failure(self) ->
|
|
187
|
+
def point_of_failure(self) -> tuple[str | None, StatusReport | None]:
|
|
188
188
|
"""Returns the most low-level failed status - most probably the point of failure
|
|
189
189
|
|
|
190
|
+
Checks for ERROR and FAILED statuses only (not DEGRADED/WARNING).
|
|
191
|
+
|
|
190
192
|
Returns:
|
|
191
|
-
(
|
|
193
|
+
tuple: (name, StatusReport) of the most low-level failed status, or (None, None)
|
|
192
194
|
"""
|
|
193
195
|
|
|
194
196
|
for k, s in reversed(self.status.items()):
|
|
195
|
-
if s ==
|
|
197
|
+
if s.status == Status.ERROR or s.status == Status.FAILED:
|
|
196
198
|
return k, s
|
|
197
199
|
return None, None
|
|
198
200
|
|
|
@@ -201,7 +203,7 @@ class HasStatuses(Manageable):
|
|
|
201
203
|
if s is None:
|
|
202
204
|
return 'OK'
|
|
203
205
|
else:
|
|
204
|
-
return f'Failed {n} ({s.
|
|
206
|
+
return f'Failed {n} ({s.message})'
|
|
205
207
|
|
|
206
208
|
def status_ok(self) -> bool:
|
|
207
209
|
"""Returns True if no failed statuses"""
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""Unified status types for serverish.
|
|
2
|
+
|
|
3
|
+
Provides a single Status enum and StatusReport dataclass used by every layer
|
|
4
|
+
of the library: diagnostics (HasStatuses), connections, monitoring, and tasks.
|
|
5
|
+
|
|
6
|
+
Status severity precedence (highest wins in aggregation):
|
|
7
|
+
FAILED > ERROR > WARNING > DEGRADED > STARTUP > SHUTDOWN > BUSY > IDLE > OK > UNKNOWN
|
|
8
|
+
|
|
9
|
+
This module replaces the former dual-system design where serverish.base had a
|
|
10
|
+
4-value StatusEnum and serverish.monitoring had a separate 10-value Status enum.
|
|
11
|
+
The two are now unified here.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from enum import Enum
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from serverish.base.datetime import dt_from_array, dt_utcnow_array
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Status(Enum):
|
|
24
|
+
"""Component/service status.
|
|
25
|
+
|
|
26
|
+
Ordered by severity (highest wins in aggregation):
|
|
27
|
+
FAILED > ERROR > WARNING > DEGRADED > STARTUP > SHUTDOWN > BUSY > IDLE > OK > UNKNOWN
|
|
28
|
+
|
|
29
|
+
Properties:
|
|
30
|
+
is_healthy: True for OK, IDLE, BUSY, DEGRADED, WARNING
|
|
31
|
+
is_operational: True for STARTUP, OK, IDLE, BUSY, DEGRADED, WARNING
|
|
32
|
+
"""
|
|
33
|
+
UNKNOWN = "unknown"
|
|
34
|
+
STARTUP = "startup"
|
|
35
|
+
OK = "ok"
|
|
36
|
+
IDLE = "idle"
|
|
37
|
+
BUSY = "busy"
|
|
38
|
+
DEGRADED = "degraded"
|
|
39
|
+
WARNING = "warning"
|
|
40
|
+
ERROR = "error"
|
|
41
|
+
SHUTDOWN = "shutdown"
|
|
42
|
+
FAILED = "failed"
|
|
43
|
+
|
|
44
|
+
def __str__(self) -> str:
|
|
45
|
+
return self.value
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def is_healthy(self) -> bool:
|
|
49
|
+
"""True for OK, IDLE, BUSY, DEGRADED, WARNING."""
|
|
50
|
+
return self in _HEALTHY_STATUSES
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def is_operational(self) -> bool:
|
|
54
|
+
"""True for STARTUP, OK, IDLE, BUSY, DEGRADED, WARNING."""
|
|
55
|
+
return self in _OPERATIONAL_STATUSES
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
_HEALTHY_STATUSES = frozenset({
|
|
59
|
+
Status.OK, Status.IDLE, Status.BUSY, Status.DEGRADED, Status.WARNING,
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
_OPERATIONAL_STATUSES = frozenset({
|
|
63
|
+
Status.STARTUP, Status.OK, Status.IDLE, Status.BUSY,
|
|
64
|
+
Status.DEGRADED, Status.WARNING,
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
_SEVERITY_RANK: dict[Status, int] = {
|
|
68
|
+
Status.UNKNOWN: 0,
|
|
69
|
+
Status.OK: 1,
|
|
70
|
+
Status.IDLE: 2,
|
|
71
|
+
Status.BUSY: 3,
|
|
72
|
+
Status.SHUTDOWN: 4,
|
|
73
|
+
Status.STARTUP: 5,
|
|
74
|
+
Status.DEGRADED: 6,
|
|
75
|
+
Status.WARNING: 7,
|
|
76
|
+
Status.ERROR: 8,
|
|
77
|
+
Status.FAILED: 9,
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass
|
|
82
|
+
class StatusReport:
|
|
83
|
+
"""Status report for any component.
|
|
84
|
+
|
|
85
|
+
Used both as a diagnostic check result (with deduce_other for deduction
|
|
86
|
+
chains in HasStatuses) and as a rich status report (with details, children,
|
|
87
|
+
and metrics for monitoring).
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
name: Component name
|
|
91
|
+
status: Current status
|
|
92
|
+
message: Optional human-readable message
|
|
93
|
+
timestamp: UTC timestamp as 7-element array [Y, M, D, h, m, s, us]
|
|
94
|
+
details: Optional dict with children reports and metrics
|
|
95
|
+
parent: Optional parent name for hierarchical grouping
|
|
96
|
+
deduce_other: If True, subsequent diagnostic checks are deduced from this result
|
|
97
|
+
"""
|
|
98
|
+
name: str
|
|
99
|
+
status: Status
|
|
100
|
+
message: str | None = None
|
|
101
|
+
timestamp: list[int] | None = None
|
|
102
|
+
details: dict[str, Any] | None = None
|
|
103
|
+
parent: str | None = None
|
|
104
|
+
deduce_other: bool = False
|
|
105
|
+
|
|
106
|
+
def __post_init__(self):
|
|
107
|
+
if self.timestamp is None:
|
|
108
|
+
self.timestamp = dt_utcnow_array()
|
|
109
|
+
|
|
110
|
+
def get_timestamp_dt(self):
|
|
111
|
+
"""Get timestamp as datetime object."""
|
|
112
|
+
if self.timestamp is None:
|
|
113
|
+
return None
|
|
114
|
+
return dt_from_array(self.timestamp)
|
|
115
|
+
|
|
116
|
+
def to_dict(self) -> dict[str, Any]:
|
|
117
|
+
"""Convert to dictionary for serialization."""
|
|
118
|
+
result: dict[str, Any] = {
|
|
119
|
+
"name": self.name,
|
|
120
|
+
"status": self.status.value,
|
|
121
|
+
"timestamp": self.timestamp,
|
|
122
|
+
}
|
|
123
|
+
if self.message:
|
|
124
|
+
result["message"] = self.message
|
|
125
|
+
if self.details:
|
|
126
|
+
result["details"] = self.details
|
|
127
|
+
if self.parent:
|
|
128
|
+
result["parent"] = self.parent
|
|
129
|
+
if self.deduce_other:
|
|
130
|
+
result["deduce_other"] = True
|
|
131
|
+
return result
|
|
132
|
+
|
|
133
|
+
@classmethod
|
|
134
|
+
def from_dict(cls, data: dict[str, Any]) -> StatusReport:
|
|
135
|
+
"""Create from dictionary.
|
|
136
|
+
|
|
137
|
+
Raises:
|
|
138
|
+
KeyError: If required fields 'name' or 'status' are missing
|
|
139
|
+
ValueError: If 'status' value is not a valid Status enum member
|
|
140
|
+
"""
|
|
141
|
+
if "name" not in data or "status" not in data:
|
|
142
|
+
missing = [k for k in ("name", "status") if k not in data]
|
|
143
|
+
raise KeyError(f"StatusReport.from_dict missing required fields: {missing}")
|
|
144
|
+
return cls(
|
|
145
|
+
name=data["name"],
|
|
146
|
+
status=Status(data["status"]),
|
|
147
|
+
message=data.get("message"),
|
|
148
|
+
timestamp=data.get("timestamp"),
|
|
149
|
+
details=data.get("details"),
|
|
150
|
+
parent=data.get("parent"),
|
|
151
|
+
deduce_other=data.get("deduce_other", False),
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
@staticmethod
|
|
155
|
+
def ok(msg: str | None = None, deduce_other: bool = True) -> StatusReport:
|
|
156
|
+
"""Create an OK status report. Defaults deduce_other=True."""
|
|
157
|
+
return StatusReport(name="", status=Status.OK, message=msg, deduce_other=deduce_other)
|
|
158
|
+
|
|
159
|
+
@staticmethod
|
|
160
|
+
def error(msg: str | None = None, deduce_other: bool = False) -> StatusReport:
|
|
161
|
+
"""Create an ERROR status report."""
|
|
162
|
+
return StatusReport(name="", status=Status.ERROR, message=msg, deduce_other=deduce_other)
|
|
163
|
+
|
|
164
|
+
@staticmethod
|
|
165
|
+
def failed(msg: str | None = None, deduce_other: bool = False) -> StatusReport:
|
|
166
|
+
"""Create a FAILED status report."""
|
|
167
|
+
return StatusReport(name="", status=Status.FAILED, message=msg, deduce_other=deduce_other)
|
|
168
|
+
|
|
169
|
+
@staticmethod
|
|
170
|
+
def unknown(msg: str | None = None, deduce_other: bool = False) -> StatusReport:
|
|
171
|
+
"""Create an UNKNOWN status report."""
|
|
172
|
+
return StatusReport(name="", status=Status.UNKNOWN, message=msg, deduce_other=deduce_other)
|
|
173
|
+
|
|
174
|
+
@staticmethod
|
|
175
|
+
def shutdown(msg: str | None = None, deduce_other: bool = False) -> StatusReport:
|
|
176
|
+
"""Create a SHUTDOWN status report."""
|
|
177
|
+
return StatusReport(name="", status=Status.SHUTDOWN, message=msg, deduce_other=deduce_other)
|
|
178
|
+
|
|
179
|
+
@staticmethod
|
|
180
|
+
def degraded(msg: str | None = None, deduce_other: bool = False) -> StatusReport:
|
|
181
|
+
"""Create a DEGRADED status report."""
|
|
182
|
+
return StatusReport(name="", status=Status.DEGRADED, message=msg, deduce_other=deduce_other)
|
|
183
|
+
|
|
184
|
+
@staticmethod
|
|
185
|
+
def warning(msg: str | None = None, deduce_other: bool = False) -> StatusReport:
|
|
186
|
+
"""Create a WARNING status report."""
|
|
187
|
+
return StatusReport(name="", status=Status.WARNING, message=msg, deduce_other=deduce_other)
|
|
188
|
+
|
|
189
|
+
@classmethod
|
|
190
|
+
def deduced(cls, source: StatusReport, msg: str | None = None) -> StatusReport:
|
|
191
|
+
"""Create a status report deduced from another result.
|
|
192
|
+
|
|
193
|
+
Used by HasStatuses diagnostic chains when a prior check's
|
|
194
|
+
deduce_other=True allows skipping subsequent checks.
|
|
195
|
+
|
|
196
|
+
Args:
|
|
197
|
+
source: The StatusReport to deduce from
|
|
198
|
+
msg: Optional message (defaults to 'Deduced')
|
|
199
|
+
"""
|
|
200
|
+
if msg is None:
|
|
201
|
+
msg = "Deduced"
|
|
202
|
+
return cls(
|
|
203
|
+
name="",
|
|
204
|
+
status=source.status,
|
|
205
|
+
message=msg,
|
|
206
|
+
deduce_other=False,
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def aggregate_status(reports: list[StatusReport]) -> Status:
|
|
211
|
+
"""Aggregate multiple status reports into single status.
|
|
212
|
+
|
|
213
|
+
Returns the most severe status from the collection, following
|
|
214
|
+
severity precedence:
|
|
215
|
+
FAILED > ERROR > WARNING > DEGRADED > STARTUP > SHUTDOWN > BUSY > IDLE > OK > UNKNOWN
|
|
216
|
+
|
|
217
|
+
Args:
|
|
218
|
+
reports: List of StatusReport instances
|
|
219
|
+
|
|
220
|
+
Returns:
|
|
221
|
+
The most severe Status from the reports, or Status.UNKNOWN if empty
|
|
222
|
+
"""
|
|
223
|
+
if not reports:
|
|
224
|
+
return Status.UNKNOWN
|
|
225
|
+
return max((r.status for r in reports), key=lambda s: _SEVERITY_RANK[s])
|
|
@@ -3,7 +3,7 @@ import logging
|
|
|
3
3
|
|
|
4
4
|
import param
|
|
5
5
|
|
|
6
|
-
from serverish.base import
|
|
6
|
+
from serverish.base import StatusReport
|
|
7
7
|
from serverish.base.asyncio_util_functions import wait_for_psce
|
|
8
8
|
from serverish.base.collector import Collector
|
|
9
9
|
from serverish.base.hasstatuses import HasStatuses
|
|
@@ -42,14 +42,14 @@ class Task(HasStatuses):
|
|
|
42
42
|
e = task.exception()
|
|
43
43
|
except asyncio.CancelledError:
|
|
44
44
|
logger.debug(f'Task {self.name} canceled')
|
|
45
|
-
self.set_status('running',
|
|
45
|
+
self.set_status('running', StatusReport.unknown(msg='Task canceled'))
|
|
46
46
|
else:
|
|
47
47
|
if e is not None:
|
|
48
48
|
logger.error(f'Task {self.name} failed: {task.exception()}', exc_info=task.exception())
|
|
49
|
-
self.set_status('running',
|
|
49
|
+
self.set_status('running', StatusReport.error(msg=f'Task failed: {task.exception()}'))
|
|
50
50
|
else:
|
|
51
51
|
logger.debug(f'Task {self.name} done')
|
|
52
|
-
self.set_status('running',
|
|
52
|
+
self.set_status('running', StatusReport.unknown(msg='Task finished'))
|
|
53
53
|
self.remove_parent()
|
|
54
54
|
|
|
55
55
|
self.task = asyncio.create_task(self.coro, name=self.name)
|
|
@@ -7,7 +7,7 @@ from typing import Iterable
|
|
|
7
7
|
|
|
8
8
|
import param
|
|
9
9
|
from serverish.base.hasstatuses import HasStatuses
|
|
10
|
-
from serverish.base.status import
|
|
10
|
+
from serverish.base.status import StatusReport
|
|
11
11
|
|
|
12
12
|
logger = logging.getLogger(__name__.rsplit('.')[-1])
|
|
13
13
|
|
|
@@ -70,7 +70,7 @@ class Connection(HasStatuses):
|
|
|
70
70
|
return f'{h}: {e}'
|
|
71
71
|
|
|
72
72
|
|
|
73
|
-
async def diagnose_dns(self) ->
|
|
73
|
+
async def diagnose_dns(self) -> StatusReport:
|
|
74
74
|
""" Diagnoses DNS connection
|
|
75
75
|
|
|
76
76
|
Returns:
|
|
@@ -79,11 +79,11 @@ class Connection(HasStatuses):
|
|
|
79
79
|
try:
|
|
80
80
|
import aiodns
|
|
81
81
|
except ImportError:
|
|
82
|
-
return
|
|
82
|
+
return StatusReport.unknown(msg='aiodns not installed, skipping DNS check')
|
|
83
83
|
|
|
84
84
|
|
|
85
85
|
if len(self.host) == 0:
|
|
86
|
-
return
|
|
86
|
+
return StatusReport.unknown(msg='No hosts to check DNS for')
|
|
87
87
|
|
|
88
88
|
resolver = aiodns.DNSResolver()
|
|
89
89
|
tasks = [self._check_host(resolver, h) for h in self.host if not self.is_ip(h)]
|
|
@@ -93,16 +93,16 @@ class Connection(HasStatuses):
|
|
|
93
93
|
failed = [res for res in results if res is not None]
|
|
94
94
|
|
|
95
95
|
if checked == 0:
|
|
96
|
-
return
|
|
96
|
+
return StatusReport.unknown(msg='No DNS names to check')
|
|
97
97
|
elif len(failed) == checked:
|
|
98
|
-
return
|
|
98
|
+
return StatusReport.error(msg=f'DNS check failed for all {checked} names: {", ".join(failed)}')
|
|
99
99
|
elif len(failed) > 0:
|
|
100
|
-
return
|
|
100
|
+
return StatusReport.ok(msg=f'DNS check ok for {checked - len(failed)} of {checked} names, '
|
|
101
101
|
f'but failed for: {", ".join(failed)}')
|
|
102
102
|
else:
|
|
103
|
-
return
|
|
103
|
+
return StatusReport.ok(msg=f'DNS check ok for all {checked} names')
|
|
104
104
|
|
|
105
|
-
async def diagnose_ping(self) ->
|
|
105
|
+
async def diagnose_ping(self) -> StatusReport:
|
|
106
106
|
"""Diagnoses ping
|
|
107
107
|
|
|
108
108
|
Returns:
|
|
@@ -118,7 +118,7 @@ class Connection(HasStatuses):
|
|
|
118
118
|
return (host, proc.returncode)
|
|
119
119
|
|
|
120
120
|
if len(self.host) == 0:
|
|
121
|
-
return
|
|
121
|
+
return StatusReport.unknown(msg='No hosts to ping')
|
|
122
122
|
|
|
123
123
|
tasks = [_ping_host(host) for host in self.host]
|
|
124
124
|
results = await asyncio.gather(*tasks)
|
|
@@ -127,12 +127,12 @@ class Connection(HasStatuses):
|
|
|
127
127
|
failed_pings = [host for host, returncode in results if returncode != 0]
|
|
128
128
|
|
|
129
129
|
if len(successful_pings) == len(self.host):
|
|
130
|
-
return
|
|
130
|
+
return StatusReport.ok(msg=f'Ping successful for all hosts: {", ".join(successful_pings)}')
|
|
131
131
|
elif len(successful_pings) > 0:
|
|
132
|
-
return
|
|
132
|
+
return StatusReport.ok(msg=f'Ping ok for {len(successful_pings)} of {len(self.host)} hosts, '
|
|
133
133
|
f'but failed for: {", ".join(failed_pings)}')
|
|
134
134
|
else:
|
|
135
|
-
return
|
|
135
|
+
return StatusReport.error(msg=f'Ping failed for all {len(self.host)} hosts: {", ".join(failed_pings)}')
|
|
136
136
|
|
|
137
137
|
def create_urls(self, protocol: str='http', path: str='') -> list[str]:
|
|
138
138
|
"""Creates list of URLs from host and port
|
|
@@ -9,7 +9,7 @@ from nats.js import JetStreamContext
|
|
|
9
9
|
import param
|
|
10
10
|
|
|
11
11
|
from serverish.connection.connection_nats import ConnectionNATS
|
|
12
|
-
from serverish.base.status import
|
|
12
|
+
from serverish.base.status import StatusReport
|
|
13
13
|
|
|
14
14
|
logger = logging.getLogger(__name__.rsplit('.')[-1])
|
|
15
15
|
|
|
@@ -148,7 +148,7 @@ class ConnectionJetStream(ConnectionNATS):
|
|
|
148
148
|
await js.update_stream(config=cfg)
|
|
149
149
|
|
|
150
150
|
|
|
151
|
-
async def diagnose_stream_config(self) ->
|
|
151
|
+
async def diagnose_stream_config(self) -> StatusReport:
|
|
152
152
|
"""Diagnoses stream configuration
|
|
153
153
|
|
|
154
154
|
Returns:
|
|
@@ -160,12 +160,12 @@ class ConnectionJetStream(ConnectionNATS):
|
|
|
160
160
|
try:
|
|
161
161
|
for s in self.streams:
|
|
162
162
|
if re.match(r'^[a-zA-Z0-9_-]+$', s) is None:
|
|
163
|
-
return
|
|
163
|
+
return StatusReport.error(msg=f'Invalid stream name {s}')
|
|
164
164
|
except TypeError:
|
|
165
|
-
return
|
|
166
|
-
return
|
|
165
|
+
return StatusReport.error(msg='Streams config invalid, can not iterate over it')
|
|
166
|
+
return StatusReport.ok(msg='Streams config valid', deduce_other=False)
|
|
167
167
|
|
|
168
|
-
async def diagnose_jetstream_init(self) ->
|
|
168
|
+
async def diagnose_jetstream_init(self) -> StatusReport:
|
|
169
169
|
"""Diagnoses JetStream connection
|
|
170
170
|
|
|
171
171
|
Returns:
|
|
@@ -173,11 +173,11 @@ class ConnectionJetStream(ConnectionNATS):
|
|
|
173
173
|
"""
|
|
174
174
|
js: JetStreamContext = self.js
|
|
175
175
|
if js is None:
|
|
176
|
-
return
|
|
176
|
+
return StatusReport.error(msg='Not initialized')
|
|
177
177
|
else:
|
|
178
|
-
return
|
|
178
|
+
return StatusReport.ok(msg='Initialized', deduce_other=False)
|
|
179
179
|
|
|
180
|
-
async def diagnose_stream_exists(self) ->
|
|
180
|
+
async def diagnose_stream_exists(self) -> StatusReport:
|
|
181
181
|
"""Diagnoses any stream existence
|
|
182
182
|
|
|
183
183
|
Returns:
|
|
@@ -185,16 +185,16 @@ class ConnectionJetStream(ConnectionNATS):
|
|
|
185
185
|
"""
|
|
186
186
|
js: JetStreamContext = self.js
|
|
187
187
|
if js is None:
|
|
188
|
-
return
|
|
188
|
+
return StatusReport.error(msg='Not initialized')
|
|
189
189
|
try:
|
|
190
190
|
info = await js.streams_info()
|
|
191
191
|
if not info:
|
|
192
|
-
return
|
|
192
|
+
return StatusReport.error(msg='Zero streams in NATS')
|
|
193
193
|
except Exception as e:
|
|
194
|
-
return
|
|
195
|
-
return
|
|
194
|
+
return StatusReport.error(msg=f'Error getting streams info: {e}')
|
|
195
|
+
return StatusReport.ok(msg='Stream(s) exist')
|
|
196
196
|
|
|
197
|
-
async def diagnose_stream_subjects(self) ->
|
|
197
|
+
async def diagnose_stream_subjects(self) -> StatusReport:
|
|
198
198
|
"""Diagnoses streams for declared subjects
|
|
199
199
|
|
|
200
200
|
Returns:
|
|
@@ -202,15 +202,15 @@ class ConnectionJetStream(ConnectionNATS):
|
|
|
202
202
|
"""
|
|
203
203
|
js: JetStreamContext = self.js
|
|
204
204
|
if js is None:
|
|
205
|
-
return
|
|
205
|
+
return StatusReport.error(msg='Not initialized')
|
|
206
206
|
if not self.declared_subjects:
|
|
207
|
-
return
|
|
207
|
+
return StatusReport.unknown(msg='No declared subjects')
|
|
208
208
|
try:
|
|
209
209
|
for subject in self.declared_subjects:
|
|
210
210
|
try:
|
|
211
211
|
await js.find_stream_name_by_subject(subject)
|
|
212
212
|
except (LookupError, TypeError) as e: # That's what nats.py throws...
|
|
213
|
-
return
|
|
213
|
+
return StatusReport.error(msg=f'Stream for {subject} not found')
|
|
214
214
|
except Exception as e:
|
|
215
|
-
return
|
|
216
|
-
return
|
|
215
|
+
return StatusReport.error(msg=f'Error getting streams info: {e}')
|
|
216
|
+
return StatusReport.ok(msg='All declared subjects have streams')
|