serverish 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- serverish-1.1.0/LICENSE +21 -0
- serverish-1.1.0/PKG-INFO +65 -0
- serverish-1.1.0/README.md +42 -0
- serverish-1.1.0/pyproject.toml +33 -0
- serverish-1.1.0/serverish/__init__.py +0 -0
- serverish-1.1.0/serverish/base/__init__.py +5 -0
- serverish-1.1.0/serverish/base/asyncio_util_functions.py +25 -0
- serverish-1.1.0/serverish/base/collector.py +35 -0
- serverish-1.1.0/serverish/base/datetime.py +67 -0
- serverish-1.1.0/serverish/base/exceptions.py +28 -0
- serverish-1.1.0/serverish/base/fifoset.py +42 -0
- serverish-1.1.0/serverish/base/hasstatuses.py +208 -0
- serverish-1.1.0/serverish/base/idmanger.py +60 -0
- serverish-1.1.0/serverish/base/manageable.py +40 -0
- serverish-1.1.0/serverish/base/singleton.py +19 -0
- serverish-1.1.0/serverish/base/status.py +132 -0
- serverish-1.1.0/serverish/base/task_manager.py +169 -0
- serverish-1.1.0/serverish/connection/__init__.py +3 -0
- serverish-1.1.0/serverish/connection/connection.py +171 -0
- serverish-1.1.0/serverish/connection/connection_jets.py +211 -0
- serverish-1.1.0/serverish/connection/connection_nats.py +200 -0
- serverish-1.1.0/serverish/messenger/__init__.py +14 -0
- serverish-1.1.0/serverish/messenger/logging.py +42 -0
- serverish-1.1.0/serverish/messenger/messenger.py +567 -0
- serverish-1.1.0/serverish/messenger/msg_callback_sub.py +98 -0
- serverish-1.1.0/serverish/messenger/msg_journal_pub.py +174 -0
- serverish-1.1.0/serverish/messenger/msg_journal_read.py +49 -0
- serverish-1.1.0/serverish/messenger/msg_progress_pub.py +349 -0
- serverish-1.1.0/serverish/messenger/msg_progress_read.py +66 -0
- serverish-1.1.0/serverish/messenger/msg_publisher.py +78 -0
- serverish-1.1.0/serverish/messenger/msg_reader.py +493 -0
- serverish-1.1.0/serverish/messenger/msg_rpc_req.py +169 -0
- serverish-1.1.0/serverish/messenger/msg_rpc_resp.py +144 -0
- serverish-1.1.0/serverish/messenger/msg_single_pub.py +47 -0
- serverish-1.1.0/serverish/messenger/msg_single_read.py +96 -0
- serverish-1.1.0/serverish/messenger/msgvalidator.py +109 -0
- serverish-1.1.0/serverish/schema/config.schema.json +25 -0
- serverish-1.1.0/serverish/schema/default.schema.json +4 -0
- serverish-1.1.0/serverish/schema/include/timestamp.schema.json +10 -0
- serverish-1.1.0/serverish/schema/journal.schema.json +59 -0
- serverish-1.1.0/serverish/schema/meta.schema.json +90 -0
- serverish-1.1.0/serverish/schema/program_current_state.schema.json +25 -0
- serverish-1.1.0/serverish/schema/program_state.schema.json +25 -0
- serverish-1.1.0/serverish/schema/progress.schema.json +59 -0
- serverish-1.1.0/serverish/schema/telemetry.schema.json +27 -0
- serverish-1.1.0/serverish/slog.py +52 -0
serverish-1.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Akond Lab
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
serverish-1.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: serverish
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: helpers for server alike projects
|
|
5
|
+
Home-page: https://github.com/AkondLab/serverish
|
|
6
|
+
Author: Mikołaj Kałuszyński
|
|
7
|
+
Requires-Python: >=3.8,<4.0
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Provides-Extra: dns
|
|
15
|
+
Provides-Extra: messenger
|
|
16
|
+
Requires-Dist: aiodns (>=3.1.1,<4.0.0) ; extra == "dns"
|
|
17
|
+
Requires-Dist: jsonschema (>=4.20.0,<5.0.0)
|
|
18
|
+
Requires-Dist: nats-py (>=2.7.2,<3.0.0) ; extra == "messenger"
|
|
19
|
+
Requires-Dist: param (>=2.1.0,<3.0.0)
|
|
20
|
+
Project-URL: Repository, https://github.com/AkondLab/serverish
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# serverish - helpers for server alike projects
|
|
24
|
+
[](https://github.com/AkondLab/serverish/actions/workflows/test-ci.yml)
|
|
25
|
+
|
|
26
|
+
## Features
|
|
27
|
+
* Component management
|
|
28
|
+
* Singletons
|
|
29
|
+
* Connections management and diagnostics
|
|
30
|
+
* NATS based Messenger
|
|
31
|
+
|
|
32
|
+
See [`doc` directory](doc/) for more documentation.
|
|
33
|
+
|
|
34
|
+
## Optional (extras) dependencies
|
|
35
|
+
In order to use Messenger, you have to add extra 'messenger' to your `project.toml` serveris dependency, e.g.:
|
|
36
|
+
```toml
|
|
37
|
+
serverish = {git="https://github.com/AkondLab/serverish.git", extras=["messenger"], branch="master"}
|
|
38
|
+
```
|
|
39
|
+
or specify dpendency with extras in the `pip` convention: `serverish[messenger]`.
|
|
40
|
+
this will install `nats-py` package.
|
|
41
|
+
|
|
42
|
+
## Plans
|
|
43
|
+
- Decorators `@messenger.sub(subject)` (Web framework style)
|
|
44
|
+
- Progress with automatic increment for known durations e.g. `pro_pub.update(completed=1, when=+20)`
|
|
45
|
+
- Pytest fixtures for NATS on CI
|
|
46
|
+
|
|
47
|
+
## Changes
|
|
48
|
+
* 1.1 Switches to `param` 2.* nad `py-nats` 1.7.*. Also, publisher may reraise exceptions or ignore them.
|
|
49
|
+
* 1.0 Changes NATS subscriptions to pull only
|
|
50
|
+
* 0.12 Extends messges metdata of `nats` section, Handles reconnection of subscriptions,
|
|
51
|
+
* 0.11 Extends, tasks, NATS connection tracking and messenger opening options (`Messenger.open`)
|
|
52
|
+
* 0.10 Adds timestamp to journal messages and extablishes schema for journal messages.
|
|
53
|
+
* 0.9.1 Makes dependency to aiodns optional.
|
|
54
|
+
* 0.9.0 Introduces Logging handler for passing standard logging messages to the Messenger.
|
|
55
|
+
* 0.8 Introduces journal publishing/reading API.
|
|
56
|
+
* 0.7 Includes progress tracking messages in the Messenger API.
|
|
57
|
+
* 0.6 introduces breaking changes in the Messenger API:
|
|
58
|
+
* Convenience functions `get_xxxxx()` returning publishers/readers are synchronous now. Remove `await`-ing for their results.
|
|
59
|
+
* Change of signature of the callback for `MsgRpcResponder.register_function` for `Rpc->None`.
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
## Credits
|
|
63
|
+
This package uses some code and ideas from the [rich](https://github.com/Textualize/rich) package (Copyright (c) 2020 Will McGugan).
|
|
64
|
+
|
|
65
|
+
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# serverish - helpers for server alike projects
|
|
2
|
+
[](https://github.com/AkondLab/serverish/actions/workflows/test-ci.yml)
|
|
3
|
+
|
|
4
|
+
## Features
|
|
5
|
+
* Component management
|
|
6
|
+
* Singletons
|
|
7
|
+
* Connections management and diagnostics
|
|
8
|
+
* NATS based Messenger
|
|
9
|
+
|
|
10
|
+
See [`doc` directory](doc/) for more documentation.
|
|
11
|
+
|
|
12
|
+
## Optional (extras) dependencies
|
|
13
|
+
In order to use Messenger, you have to add extra 'messenger' to your `project.toml` serveris dependency, e.g.:
|
|
14
|
+
```toml
|
|
15
|
+
serverish = {git="https://github.com/AkondLab/serverish.git", extras=["messenger"], branch="master"}
|
|
16
|
+
```
|
|
17
|
+
or specify dpendency with extras in the `pip` convention: `serverish[messenger]`.
|
|
18
|
+
this will install `nats-py` package.
|
|
19
|
+
|
|
20
|
+
## Plans
|
|
21
|
+
- Decorators `@messenger.sub(subject)` (Web framework style)
|
|
22
|
+
- Progress with automatic increment for known durations e.g. `pro_pub.update(completed=1, when=+20)`
|
|
23
|
+
- Pytest fixtures for NATS on CI
|
|
24
|
+
|
|
25
|
+
## Changes
|
|
26
|
+
* 1.1 Switches to `param` 2.* nad `py-nats` 1.7.*. Also, publisher may reraise exceptions or ignore them.
|
|
27
|
+
* 1.0 Changes NATS subscriptions to pull only
|
|
28
|
+
* 0.12 Extends messges metdata of `nats` section, Handles reconnection of subscriptions,
|
|
29
|
+
* 0.11 Extends, tasks, NATS connection tracking and messenger opening options (`Messenger.open`)
|
|
30
|
+
* 0.10 Adds timestamp to journal messages and extablishes schema for journal messages.
|
|
31
|
+
* 0.9.1 Makes dependency to aiodns optional.
|
|
32
|
+
* 0.9.0 Introduces Logging handler for passing standard logging messages to the Messenger.
|
|
33
|
+
* 0.8 Introduces journal publishing/reading API.
|
|
34
|
+
* 0.7 Includes progress tracking messages in the Messenger API.
|
|
35
|
+
* 0.6 introduces breaking changes in the Messenger API:
|
|
36
|
+
* Convenience functions `get_xxxxx()` returning publishers/readers are synchronous now. Remove `await`-ing for their results.
|
|
37
|
+
* Change of signature of the callback for `MsgRpcResponder.register_function` for `Rpc->None`.
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
## Credits
|
|
41
|
+
This package uses some code and ideas from the [rich](https://github.com/Textualize/rich) package (Copyright (c) 2020 Will McGugan).
|
|
42
|
+
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
[tool.poetry]
|
|
2
|
+
name = "serverish"
|
|
3
|
+
version = "1.1.0"
|
|
4
|
+
description = "helpers for server alike projects"
|
|
5
|
+
authors = ["Mikołaj Kałuszyński", "MMME team"]
|
|
6
|
+
readme = "README.md"
|
|
7
|
+
repository = "https://github.com/AkondLab/serverish"
|
|
8
|
+
include = ["serverish/schema/*"]
|
|
9
|
+
|
|
10
|
+
[tool.poetry.dependencies]
|
|
11
|
+
python = "^3.8"
|
|
12
|
+
aiodns = {version="^3.1.1", optional=true}
|
|
13
|
+
param = "^2.1.0"
|
|
14
|
+
nats-py = { version = "^2.7.2", optional= true}
|
|
15
|
+
jsonschema = "^4.20.0"
|
|
16
|
+
|
|
17
|
+
[tool.poetry.extras]
|
|
18
|
+
messenger = ["nats-py"]
|
|
19
|
+
dns = ["aiodns"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
[tool.poetry.group.dev.dependencies]
|
|
23
|
+
pytest = "^7.4.0"
|
|
24
|
+
pytest-asyncio = "^0.21.0"
|
|
25
|
+
nats-py = "^2.6.0"
|
|
26
|
+
jupyter = "*"
|
|
27
|
+
rich = "^13.5.2"
|
|
28
|
+
aiodns = "^3.1.1"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
[build-system]
|
|
32
|
+
requires = ["poetry-core"]
|
|
33
|
+
build-backend = "poetry.core.masonry.api"
|
|
File without changes
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from contextlib import suppress
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
async def wait_for_psce(fut, timeout):
|
|
6
|
+
"""
|
|
7
|
+
Meaning: psce - prevent silent cancel error \n
|
|
8
|
+
This is a changed method 'wait_for()' from library 'asyncio' that prevents 'CancelledError' from being ignored
|
|
9
|
+
while the internal task is finished. \n
|
|
10
|
+
Description of the problem: the base method 'wait_for()' also works in such a way that when the inner task
|
|
11
|
+
is finished or there is no place for it to stop, function 'wait_for()' will return the result even if the inner
|
|
12
|
+
task is canceled, so the 'CancelledError' will not be propagated higher. This method prevents this situation.
|
|
13
|
+
|
|
14
|
+
:param fut: future oc coroutine
|
|
15
|
+
:param timeout: timeout, czn be float or int number
|
|
16
|
+
"""
|
|
17
|
+
task = asyncio.ensure_future(fut)
|
|
18
|
+
try:
|
|
19
|
+
return await asyncio.shield(asyncio.wait_for(task, timeout=timeout))
|
|
20
|
+
except asyncio.CancelledError:
|
|
21
|
+
task.cancel()
|
|
22
|
+
with suppress(asyncio.CancelledError):
|
|
23
|
+
await task
|
|
24
|
+
raise
|
|
25
|
+
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
import param
|
|
4
|
+
|
|
5
|
+
from serverish.base.manageable import Manageable
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__.rsplit('.')[-1])
|
|
8
|
+
|
|
9
|
+
class Collector(Manageable):
|
|
10
|
+
"""Has manageable children"""
|
|
11
|
+
# children: list[Manageable] = None
|
|
12
|
+
children_by_name: dict[str, Manageable] = param.Dict(default=dict(), allow_None=False, doc='Children by name')
|
|
13
|
+
children_names: dict[Manageable, str] = param.Dict(default=dict(), allow_None=False, doc='Names by child')
|
|
14
|
+
|
|
15
|
+
def ensure_parenting(self, child: Manageable):
|
|
16
|
+
try:
|
|
17
|
+
name = self.children_names[child]
|
|
18
|
+
except KeyError:
|
|
19
|
+
name = child.name
|
|
20
|
+
self.children_by_name[name] = child
|
|
21
|
+
self.children_names[child] = name
|
|
22
|
+
child.parent = self
|
|
23
|
+
|
|
24
|
+
def remove_child(self, child: Manageable):
|
|
25
|
+
try:
|
|
26
|
+
name = self.children_names[child]
|
|
27
|
+
except KeyError:
|
|
28
|
+
name = child.name
|
|
29
|
+
del self.children_by_name[name]
|
|
30
|
+
del self.children_names[child]
|
|
31
|
+
child.parent = None
|
|
32
|
+
|
|
33
|
+
@property
|
|
34
|
+
def children(self):
|
|
35
|
+
return self.children_by_name.values()
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
""" Those functions are used to convert datetime objects to array representation
|
|
2
|
+
(7-element list, similar to time tuple) and vice versa.
|
|
3
|
+
|
|
4
|
+
This representation is chosen for serializing datetime to JSON as fast and easy to process.
|
|
5
|
+
We are using list instead of tuple for smooth conversion to/from JSON.
|
|
6
|
+
|
|
7
|
+
The meaning of the elements of the array is as follows:
|
|
8
|
+
[year, month, day, hour, minute, second, microsecond]
|
|
9
|
+
Time is in UTC.
|
|
10
|
+
This is compatible with the datetime constructor, e.g.:
|
|
11
|
+
dt = datetime(*array, tzinfo=timezone.utc)
|
|
12
|
+
but you have also the convenience function:
|
|
13
|
+
dt = dt_from_array(array)
|
|
14
|
+
to do the same preserving None.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from datetime import datetime
|
|
20
|
+
from datetime import timezone
|
|
21
|
+
from typing import Sequence
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def dt_to_array(dt: datetime | None) -> list | None:
|
|
25
|
+
if dt is None:
|
|
26
|
+
return None
|
|
27
|
+
else:
|
|
28
|
+
dt = dt.astimezone(timezone.utc)
|
|
29
|
+
return [dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.microsecond]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def dt_from_array(t: Sequence | None) -> datetime | None:
|
|
33
|
+
if not t:
|
|
34
|
+
return None
|
|
35
|
+
try:
|
|
36
|
+
return datetime(*t, tzinfo=timezone.utc)
|
|
37
|
+
except TypeError: # Old 9-position format?
|
|
38
|
+
return datetime(*t[:7])
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def dt_ensure_array(dt: datetime | Sequence | float | None) -> list | None:
|
|
42
|
+
"""Supports datetime, array, float (POSIX timestamp) and None"""
|
|
43
|
+
if dt is None:
|
|
44
|
+
return None
|
|
45
|
+
elif isinstance(dt, datetime):
|
|
46
|
+
return dt_to_array(dt)
|
|
47
|
+
elif isinstance(dt, float):
|
|
48
|
+
return dt_to_array(datetime.fromtimestamp(dt, timezone.utc))
|
|
49
|
+
else:
|
|
50
|
+
return list(dt)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def dt_ensure_datetime(dt: datetime | Sequence | float | None) -> datetime | None:
|
|
54
|
+
"""Supports datetime, array, float (POSIX timestamp) and None"""
|
|
55
|
+
if isinstance(dt, datetime) or dt is None:
|
|
56
|
+
return dt
|
|
57
|
+
elif isinstance(dt, float):
|
|
58
|
+
return datetime.fromtimestamp(dt, timezone.utc)
|
|
59
|
+
else:
|
|
60
|
+
return dt_from_array(dt)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def dt_utcnow_array() -> list:
|
|
64
|
+
dt = datetime.now(timezone.utc)
|
|
65
|
+
return dt_to_array(dt)
|
|
66
|
+
|
|
67
|
+
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
class MessengerCannotConnect(Exception):
|
|
2
|
+
pass
|
|
3
|
+
|
|
4
|
+
class MessengerNotConnected(Exception):
|
|
5
|
+
pass
|
|
6
|
+
|
|
7
|
+
class MessengerReaderStopped(Exception):
|
|
8
|
+
pass
|
|
9
|
+
|
|
10
|
+
class MessengerRequestNoResponse(Exception):
|
|
11
|
+
pass
|
|
12
|
+
class MessengerRequestNoResponders(MessengerRequestNoResponse):
|
|
13
|
+
pass
|
|
14
|
+
|
|
15
|
+
class MessengerRequestJetStreamSubject(MessengerRequestNoResponse):
|
|
16
|
+
def __init__(self, subject:str) -> None:
|
|
17
|
+
super().__init__(f'Subject {subject} probably declared in JestStream stream. '
|
|
18
|
+
f'Use pure NATS core subjects for RPC')
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class MessengerRequestCanceled(MessengerRequestNoResponse):
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
class MessengerRequestTimeout(MessengerRequestNoResponse):
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
class MessengerRequestNoResultYet(MessengerRequestNoResponse):
|
|
28
|
+
pass
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# This is generic class implementing a set of specified capacity.
|
|
2
|
+
# When the capacity is reach during the insertion of a new element,
|
|
3
|
+
# the oldest element is removed from the set.
|
|
4
|
+
|
|
5
|
+
from typing import TypeVar, Generic
|
|
6
|
+
from collections import deque
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
# The type T of the set elements:
|
|
10
|
+
T = TypeVar('T')
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class FifoSet(Generic[T]):
|
|
14
|
+
def __init__(self, capacity: int):
|
|
15
|
+
self.capacity = capacity
|
|
16
|
+
self.cache = set()
|
|
17
|
+
self.order = deque()
|
|
18
|
+
|
|
19
|
+
def __contains__(self, item: T) -> bool:
|
|
20
|
+
return item in self.cache
|
|
21
|
+
|
|
22
|
+
def __len__(self) -> int:
|
|
23
|
+
return len(self.cache)
|
|
24
|
+
|
|
25
|
+
def __iter__(self):
|
|
26
|
+
return iter(self.cache)
|
|
27
|
+
|
|
28
|
+
def add(self, item: T):
|
|
29
|
+
if item in self.cache:
|
|
30
|
+
return
|
|
31
|
+
if len(self.cache) >= self.capacity:
|
|
32
|
+
self.cache.remove(self.order.popleft())
|
|
33
|
+
self.cache.add(item)
|
|
34
|
+
self.order.append(item)
|
|
35
|
+
|
|
36
|
+
def remove(self, item: T):
|
|
37
|
+
self.cache.remove(item)
|
|
38
|
+
self.order.remove(item)
|
|
39
|
+
|
|
40
|
+
def clear(self):
|
|
41
|
+
self.cache.clear()
|
|
42
|
+
self.order.clear()
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import asyncio
|
|
3
|
+
import logging
|
|
4
|
+
from typing import Callable, Awaitable, Union
|
|
5
|
+
|
|
6
|
+
import param
|
|
7
|
+
|
|
8
|
+
from serverish.base.collector import Collector
|
|
9
|
+
from serverish.base.manageable import Manageable
|
|
10
|
+
from serverish.base.status import Status, StatusEnum
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__.rsplit('.')[-1])
|
|
13
|
+
|
|
14
|
+
CheckMethodType = Callable[[], Union[Awaitable[Status], Status]]
|
|
15
|
+
|
|
16
|
+
class HasStatuses(Manageable):
|
|
17
|
+
"""Controls the resource
|
|
18
|
+
|
|
19
|
+
Parameters
|
|
20
|
+
----------
|
|
21
|
+
status : dict[str, Status], optional
|
|
22
|
+
Statuses of the resource, by default None
|
|
23
|
+
|
|
24
|
+
"""
|
|
25
|
+
@staticmethod
|
|
26
|
+
def diagnose_dummy_ok() -> Status:
|
|
27
|
+
"""Dummy diagnosis, returns StatusEnum.ok"""
|
|
28
|
+
return Status.new_ok(msg='Dummy OK')
|
|
29
|
+
|
|
30
|
+
status = param.Dict(default={}, doc='A dictionary to hold status information')
|
|
31
|
+
|
|
32
|
+
def __init__(self, name: str = None, parent: Collector = None, **kwargs) -> None:
|
|
33
|
+
self.check_methods: dict[str, CheckMethodType] = {'main': self.diagnose_dummy_ok}
|
|
34
|
+
|
|
35
|
+
super().__init__(name, parent, **kwargs)
|
|
36
|
+
|
|
37
|
+
def set_check_methods(self, **methods: CheckMethodType) -> None:
|
|
38
|
+
"""Sets check methods
|
|
39
|
+
|
|
40
|
+
Methods are called in order, values of later methods may be deduced from earlier ones:
|
|
41
|
+
if some method returns StatusEnum.ok, or StatusEnum.na, we deduce that the result of all later methods
|
|
42
|
+
is the same.
|
|
43
|
+
E.g. if 'ping' is OK, we deduce that 'dns' is OK too.
|
|
44
|
+
One can force all diagnotics to be run by setting no_deduce=True in diagnose() call.
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
Args:
|
|
48
|
+
**methods (dict[str, CheckMethodType]): Dict of asynchronous methods
|
|
49
|
+
"""
|
|
50
|
+
self.check_methods = methods
|
|
51
|
+
|
|
52
|
+
def add_check_methods(self, at_beginning: bool,
|
|
53
|
+
**methods: CheckMethodType) -> None:
|
|
54
|
+
"""Adds check methods
|
|
55
|
+
|
|
56
|
+
Methods are called in order, values of later methods may be deduced from earlier ones:
|
|
57
|
+
if some method returns StatusEnum.ok, or StatusEnum.na, we deduce that the result of all later methods
|
|
58
|
+
is the same.
|
|
59
|
+
E.g. if 'ping' is OK, we deduce that 'dns' is OK too.
|
|
60
|
+
One can force all diagnotics to be run by setting no_deduce=True in diagnose() call.
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
at_beginning (bool): If True, adds methods at the beginning of the list.
|
|
64
|
+
**methods (dict[str, Callable]): Dict of asynchronous methods to add
|
|
65
|
+
"""
|
|
66
|
+
if at_beginning:
|
|
67
|
+
self.check_methods = {**methods, **self.check_methods}
|
|
68
|
+
else:
|
|
69
|
+
self.check_methods = {**self.check_methods, **methods}
|
|
70
|
+
|
|
71
|
+
def set_status(self, key: str, value: Status | None) -> None:
|
|
72
|
+
"""Sets status of the resource
|
|
73
|
+
|
|
74
|
+
A resource can have multiple statuses, e.g. 'main', 'dns', 'http', etc.
|
|
75
|
+
Args:
|
|
76
|
+
key (str): Status name
|
|
77
|
+
value (Status): Status object
|
|
78
|
+
"""
|
|
79
|
+
if self.status is None:
|
|
80
|
+
self.status = {}
|
|
81
|
+
if value is None:
|
|
82
|
+
self.status.pop(key, None)
|
|
83
|
+
else:
|
|
84
|
+
self.status[key] = value
|
|
85
|
+
|
|
86
|
+
async def diagnose(self, no_deduce: bool = False) -> dict[str, Status]:
|
|
87
|
+
"""Diagnoses the object
|
|
88
|
+
|
|
89
|
+
Diagnose by calling check methods and setting statuses.
|
|
90
|
+
Methods are called in order, values of later methods may be deduced from earlier ones:
|
|
91
|
+
if some method returns StatusEnum.ok, or StatusEnum.na, we deduce that the result of all later methods
|
|
92
|
+
is the same.
|
|
93
|
+
E.g. if 'ping' is OK, we deduce that 'dns' is OK too.
|
|
94
|
+
|
|
95
|
+
One can force all diagnotics to be run by setting no_deduce=True.
|
|
96
|
+
|
|
97
|
+
Args:
|
|
98
|
+
no_deduce (bool, optional): If True, don't deduce statuses from other statuses. Defaults to False.
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
async def calc_state(m):
|
|
102
|
+
if asyncio.iscoroutinefunction(m):
|
|
103
|
+
return await m()
|
|
104
|
+
else:
|
|
105
|
+
return m()
|
|
106
|
+
|
|
107
|
+
if no_deduce:
|
|
108
|
+
checking_tasks = [calc_state(m) for m in self.check_methods.values()]
|
|
109
|
+
res = await asyncio.gather(*checking_tasks)
|
|
110
|
+
else:
|
|
111
|
+
res = []
|
|
112
|
+
deduced_status: Status | None = None
|
|
113
|
+
deduced_name: str | None = None
|
|
114
|
+
for n, m in self.check_methods.items():
|
|
115
|
+
if deduced_status is not None:
|
|
116
|
+
r = Status.deduced(deduced_status, f'Deduced from {deduced_name}')
|
|
117
|
+
else:
|
|
118
|
+
r = await calc_state(m)
|
|
119
|
+
if r.deduce_other:
|
|
120
|
+
deduced_status = r
|
|
121
|
+
deduced_name = n
|
|
122
|
+
res.append(r)
|
|
123
|
+
return {n: r for n, r in zip(self.check_methods.keys(), res)}
|
|
124
|
+
|
|
125
|
+
def diagnose_sync(self, no_deduce: bool = False) -> dict[str, Status]:
|
|
126
|
+
"""Diagnoses the object skipping async methods
|
|
127
|
+
|
|
128
|
+
Diagnose by calling check methods and setting statuses.
|
|
129
|
+
Methods are called in order, values of later methods may be deduced from earlier ones:
|
|
130
|
+
if some method returns StatusEnum.ok, or StatusEnum.na, we deduce that the result of all later methods
|
|
131
|
+
is the same.
|
|
132
|
+
E.g. if 'ping' is OK, we deduce that 'dns' is OK too.
|
|
133
|
+
|
|
134
|
+
One can force all diagnotics to be run by setting no_deduce=True.
|
|
135
|
+
|
|
136
|
+
Args:
|
|
137
|
+
no_deduce (bool, optional): If True, don't deduce statuses from other statuses. Defaults to False.
|
|
138
|
+
"""
|
|
139
|
+
|
|
140
|
+
def calc_state(m):
|
|
141
|
+
if asyncio.iscoroutinefunction(m):
|
|
142
|
+
return Status.new_na(msg='Skipped async test')
|
|
143
|
+
else:
|
|
144
|
+
return m()
|
|
145
|
+
|
|
146
|
+
if no_deduce:
|
|
147
|
+
res = [calc_state(m) for m in self.check_methods.values()]
|
|
148
|
+
else:
|
|
149
|
+
res = []
|
|
150
|
+
deduced_status: Status | None = None
|
|
151
|
+
deduced_name: str | None = None
|
|
152
|
+
for n, m in self.check_methods.items():
|
|
153
|
+
if deduced_status is not None:
|
|
154
|
+
r = Status.deduced(deduced_status, f'Deduced from {deduced_name}')
|
|
155
|
+
else:
|
|
156
|
+
r = calc_state(m)
|
|
157
|
+
if r.deduce_other:
|
|
158
|
+
deduced_status = r
|
|
159
|
+
deduced_name = n
|
|
160
|
+
res.append(r)
|
|
161
|
+
return {n: r for n, r in zip(self.check_methods.keys(), res)}
|
|
162
|
+
|
|
163
|
+
async def update_statuses(self, no_deduce = False) -> None:
|
|
164
|
+
"""Updates statuses of the resource
|
|
165
|
+
|
|
166
|
+
Args:
|
|
167
|
+
no_deduce (bool, optional): If True, don't deduce statuses from other statuses. Defaults to False.
|
|
168
|
+
|
|
169
|
+
Calls diagnose() and sets statuses
|
|
170
|
+
"""
|
|
171
|
+
statuses = await self.diagnose(no_deduce=no_deduce)
|
|
172
|
+
for n, s in statuses.items():
|
|
173
|
+
self.set_status(n, s)
|
|
174
|
+
|
|
175
|
+
def update_sync_statuses(self, no_deduce = False) -> None:
|
|
176
|
+
"""Updates statuses of the resource skipping async methods
|
|
177
|
+
|
|
178
|
+
Args:
|
|
179
|
+
no_deduce (bool, optional): If True, don't deduce statuses from other statuses. Defaults to False.
|
|
180
|
+
|
|
181
|
+
Calls diagnose() and sets statuses
|
|
182
|
+
"""
|
|
183
|
+
statuses = self.diagnose_sync(no_deduce=no_deduce)
|
|
184
|
+
for n, s in statuses.items():
|
|
185
|
+
self.set_status(n, s)
|
|
186
|
+
|
|
187
|
+
def point_of_failure(self) -> (str | None, Status | None):
|
|
188
|
+
"""Returns the most low-level failed status - most probably the point of failure
|
|
189
|
+
|
|
190
|
+
Returns:
|
|
191
|
+
(str, Status): Name and status of the most low-level failed status
|
|
192
|
+
"""
|
|
193
|
+
|
|
194
|
+
for k, s in reversed(self.status.items()):
|
|
195
|
+
if s == StatusEnum.fail:
|
|
196
|
+
return k, s
|
|
197
|
+
return None, None
|
|
198
|
+
|
|
199
|
+
def format_status(self) -> str:
|
|
200
|
+
n, s = self.point_of_failure()
|
|
201
|
+
if s is None:
|
|
202
|
+
return 'OK'
|
|
203
|
+
else:
|
|
204
|
+
return f'Failed {n} ({s.msg})'
|
|
205
|
+
|
|
206
|
+
def status_ok(self) -> bool:
|
|
207
|
+
"""Returns True if no failed statuses"""
|
|
208
|
+
return self.point_of_failure()[1] is None
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
from serverish.base.singleton import Singleton
|
|
4
|
+
|
|
5
|
+
logger = logging.getLogger(__name__.rsplit('.')[-1])
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def gen_id(name: str):
|
|
9
|
+
"""Generates unique ID for the prefix name"""
|
|
10
|
+
ids = IdManager()
|
|
11
|
+
return ids.get_id(name)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def gen_uid(name: str, length: int = 10):
|
|
15
|
+
"""Generates "unique" hex ID of given length with the prefix name
|
|
16
|
+
|
|
17
|
+
Uses uuid4 and URL safe base64 encoding to generate unique ID
|
|
18
|
+
"""
|
|
19
|
+
return IdManager.get_uid(name, length)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class IdManager(Singleton):
|
|
23
|
+
"""Manages unique IDs"""
|
|
24
|
+
ids_counters = {}
|
|
25
|
+
|
|
26
|
+
def get_id(self, name: str) -> str:
|
|
27
|
+
"""Returns unique ID for the prefix name
|
|
28
|
+
|
|
29
|
+
name may contain format specifiers, e.g. 'obj-{:04d}', if present, counter will
|
|
30
|
+
be formatted with the specifier.
|
|
31
|
+
If there is no specifier, if counter is zero, `name` will be returned, otherwise
|
|
32
|
+
`name-<counter>` will be returned.
|
|
33
|
+
"""
|
|
34
|
+
if name not in self.ids_counters:
|
|
35
|
+
self.ids_counters[name] = 0
|
|
36
|
+
self.ids_counters[name] += 1
|
|
37
|
+
|
|
38
|
+
if '{' not in name:
|
|
39
|
+
if self.ids_counters[name] == 1:
|
|
40
|
+
ret = name
|
|
41
|
+
else:
|
|
42
|
+
ret = f'{name}-{self.ids_counters[name]}'
|
|
43
|
+
else:
|
|
44
|
+
ret = name.format(self.ids_counters[name])
|
|
45
|
+
return ret
|
|
46
|
+
|
|
47
|
+
def get_ids(self, name: str, count: int) -> list[str]:
|
|
48
|
+
"""Returns generator of unique IDs for the name"""
|
|
49
|
+
for _ in range(count):
|
|
50
|
+
yield self.get_id(name)
|
|
51
|
+
|
|
52
|
+
@staticmethod
|
|
53
|
+
def get_uid(name: str, length: int = 10) -> str:
|
|
54
|
+
"""Returns "unique" hex ID of given length with the prefix name
|
|
55
|
+
|
|
56
|
+
Uses uuid4 and URL safe base64 encoding to generate unique ID
|
|
57
|
+
"""
|
|
58
|
+
import uuid
|
|
59
|
+
import base64
|
|
60
|
+
return f'{name}{base64.urlsafe_b64encode(uuid.uuid4().bytes).decode()[:length]}'
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import logging
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
import param
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from serverish.base.collector import Collector
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__.rsplit('.')[-1])
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Manageable(param.Parameterized):
|
|
15
|
+
""" Manageable object
|
|
16
|
+
|
|
17
|
+
Manageable object is an object that can be managed by a 'parent' collector."""
|
|
18
|
+
|
|
19
|
+
def __init__(self, name: str = None, parent: Collector = None, **kwargs) -> None:
|
|
20
|
+
self.parent: Collector = parent
|
|
21
|
+
from serverish.base.idmanger import gen_uid, gen_id
|
|
22
|
+
if name is None:
|
|
23
|
+
name = gen_uid(f'{self.__class__.__name__}_')
|
|
24
|
+
else:
|
|
25
|
+
name = gen_id(name)
|
|
26
|
+
super().__init__(name=name, **kwargs)
|
|
27
|
+
if parent is not None:
|
|
28
|
+
self.parent.ensure_parenting(self)
|
|
29
|
+
|
|
30
|
+
def remove_parent(self):
|
|
31
|
+
"""Removes parent"""
|
|
32
|
+
if self.parent is not None:
|
|
33
|
+
self.parent.remove_child(self)
|
|
34
|
+
self.parent = None
|
|
35
|
+
|
|
36
|
+
def set_parent(self, parent: Collector):
|
|
37
|
+
"""Sets parent"""
|
|
38
|
+
self.remove_parent()
|
|
39
|
+
self.parent = parent
|
|
40
|
+
self.parent.ensure_parenting(self)
|