moat-kv-akumuli 0.8.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moat-kv-akumuli-0.8.3/.gitignore +8 -0
- moat-kv-akumuli-0.8.3/.pylintrc +3 -0
- moat-kv-akumuli-0.8.3/Makefile +14 -0
- moat-kv-akumuli-0.8.3/PKG-INFO +25 -0
- moat-kv-akumuli-0.8.3/README.rst +7 -0
- moat-kv-akumuli-0.8.3/TODO.rst +3 -0
- moat-kv-akumuli-0.8.3/USAGE.rst +92 -0
- moat-kv-akumuli-0.8.3/moat/kv/akumuli/__init__.py +0 -0
- moat-kv-akumuli-0.8.3/moat/kv/akumuli/_config.yaml +7 -0
- moat-kv-akumuli-0.8.3/moat/kv/akumuli/_main.py +177 -0
- moat-kv-akumuli-0.8.3/moat/kv/akumuli/model.py +242 -0
- moat-kv-akumuli-0.8.3/moat/kv/akumuli/task.py +71 -0
- moat-kv-akumuli-0.8.3/moat-kv-akumuli@.service +22 -0
- moat-kv-akumuli-0.8.3/moat_kv_akumuli.egg-info/PKG-INFO +25 -0
- moat-kv-akumuli-0.8.3/moat_kv_akumuli.egg-info/SOURCES.txt +21 -0
- moat-kv-akumuli-0.8.3/moat_kv_akumuli.egg-info/dependency_links.txt +1 -0
- moat-kv-akumuli-0.8.3/moat_kv_akumuli.egg-info/requires.txt +1 -0
- moat-kv-akumuli-0.8.3/moat_kv_akumuli.egg-info/top_level.txt +1 -0
- moat-kv-akumuli-0.8.3/pyproject.toml +98 -0
- moat-kv-akumuli-0.8.3/setup.cfg +4 -0
- moat-kv-akumuli-0.8.3/tests/__init__.py +0 -0
- moat-kv-akumuli-0.8.3/tests/conftest.py +5 -0
- moat-kv-akumuli-0.8.3/tests/test_akumuli.py +50 -0
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#!/usr/bin/make -f
|
|
2
|
+
|
|
3
|
+
PACKAGE = moat-kv-akumuli
|
|
4
|
+
MAKEINCL ?= $(shell python3 -mmoat src path)/make/py
|
|
5
|
+
|
|
6
|
+
ifneq ($(wildcard $(MAKEINCL)),)
|
|
7
|
+
include $(MAKEINCL)
|
|
8
|
+
# availabe via https://github.com/moat-src
|
|
9
|
+
|
|
10
|
+
else
|
|
11
|
+
%:
|
|
12
|
+
@echo "Please fix 'python3 -mmoat src path'."
|
|
13
|
+
@exit 1
|
|
14
|
+
endif
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: moat-kv-akumuli
|
|
3
|
+
Version: 0.8.3
|
|
4
|
+
Summary: Akumuli storage connector for MoaT-KV
|
|
5
|
+
Author-email: Matthias Urlichs <matthias@urlichs.de>
|
|
6
|
+
Project-URL: homepage, https://m-o-a-t.org
|
|
7
|
+
Project-URL: repository, https://github.com/M-o-a-T/moat-kv-akumuli
|
|
8
|
+
Keywords: MoaT
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Information Technology
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Database
|
|
15
|
+
Classifier: Topic :: Home Automation
|
|
16
|
+
Requires-Python: >=3.8
|
|
17
|
+
Description-Content-Type: text/x-rst
|
|
18
|
+
|
|
19
|
+
===========
|
|
20
|
+
DistAkumuli
|
|
21
|
+
===========
|
|
22
|
+
|
|
23
|
+
DistAkumuli is a simple link between Akumuli and DistKV.
|
|
24
|
+
|
|
25
|
+
It will update an Akumuli series whenever a DistKV value changes.
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
=====================
|
|
2
|
+
Using MoaT-KV-Akumuli
|
|
3
|
+
=====================
|
|
4
|
+
|
|
5
|
+
Run "moat kv akumuli monitor server" to connect to the specified server.
|
|
6
|
+
|
|
7
|
+
See "moat util cfg kv.akumuli" for configuration options.
|
|
8
|
+
|
|
9
|
+
Data structure
|
|
10
|
+
==============
|
|
11
|
+
|
|
12
|
+
On disk, the path to the port is ".moat kv akumuli SERVER NAME…" by
|
|
13
|
+
default. All attributes are also looked up in the higher nodes, so you can
|
|
14
|
+
set per-type values easily.
|
|
15
|
+
|
|
16
|
+
Attributes
|
|
17
|
+
++++++++++
|
|
18
|
+
|
|
19
|
+
* server: a dict with host and port. Set by ``moat kv akumuli server``.
|
|
20
|
+
|
|
21
|
+
* metric: the Akumuli metric to use. Inherited.
|
|
22
|
+
|
|
23
|
+
* keys: a dict with keys that a value shall have. Inherited.
|
|
24
|
+
|
|
25
|
+
* src: An array. The path to the Moat-KV value which shall be stored.
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
Command line
|
|
29
|
+
============
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
.. program:: moat kv akumuli
|
|
33
|
+
|
|
34
|
+
The main entry point for this extension.
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
.. program:: moat kv akumuli set
|
|
38
|
+
|
|
39
|
+
Print or modify port settings.
|
|
40
|
+
|
|
41
|
+
This is a shortcut for ``… attr`` that evaluates various attributes and
|
|
42
|
+
lets you easily change more than one at a time.
|
|
43
|
+
|
|
44
|
+
.. option:: -m, --metric METRIC
|
|
45
|
+
|
|
46
|
+
Set the metric to use in this subtree.
|
|
47
|
+
|
|
48
|
+
This option cannot be used on top or server level.
|
|
49
|
+
|
|
50
|
+
.. option:: -k, --key name=value
|
|
51
|
+
|
|
52
|
+
Add a key. An empty value deletes the key.
|
|
53
|
+
|
|
54
|
+
This option can be used more than once.
|
|
55
|
+
|
|
56
|
+
.. option:: -s, --src "source path"
|
|
57
|
+
|
|
58
|
+
Set the value to record.
|
|
59
|
+
|
|
60
|
+
This option cannot be used on top or server level.
|
|
61
|
+
|
|
62
|
+
.. option:: -e, --eval
|
|
63
|
+
|
|
64
|
+
The value of 'src' shall be evaluated.
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
.. option:: path
|
|
68
|
+
|
|
69
|
+
The path to the node to be modified.
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
.. program:: moat kv akumuli list
|
|
73
|
+
|
|
74
|
+
Lists the names below of a subtree.
|
|
75
|
+
|
|
76
|
+
.. option:: path
|
|
77
|
+
|
|
78
|
+
The path to the node to be shown.
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
.. program:: moat kv akumuli get
|
|
82
|
+
|
|
83
|
+
Displays the data at a node.
|
|
84
|
+
|
|
85
|
+
.. option:: -r, --recursive
|
|
86
|
+
|
|
87
|
+
Lists the subtree starting here.
|
|
88
|
+
|
|
89
|
+
.. option:: path
|
|
90
|
+
|
|
91
|
+
The path to the node to be shown.
|
|
92
|
+
|
|
File without changes
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
# command line interface
|
|
2
|
+
|
|
3
|
+
import asyncclick as click
|
|
4
|
+
from asyncakumuli import DS
|
|
5
|
+
|
|
6
|
+
from moat.util import yprint, attrdict, NotGiven, as_service, P, attr_args
|
|
7
|
+
from moat.kv.data import node_attr, data_get
|
|
8
|
+
from moat.kv.obj.command import std_command
|
|
9
|
+
|
|
10
|
+
from .model import AkumuliRoot
|
|
11
|
+
|
|
12
|
+
import logging
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
async def _prepare_cli(obj):
|
|
18
|
+
obj.data = await AkumuliRoot.as_handler(obj.client)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
cli = std_command(
|
|
22
|
+
click,
|
|
23
|
+
name="server",
|
|
24
|
+
sub_base=None,
|
|
25
|
+
sub_name=NotGiven,
|
|
26
|
+
id_name=None,
|
|
27
|
+
prepare=_prepare_cli,
|
|
28
|
+
aux=(
|
|
29
|
+
click.option("-h", "--host", help="Host name of this server.", type=str),
|
|
30
|
+
click.option("-p", "--port", help="Port of this server.", type=int),
|
|
31
|
+
click.option("-t", "--topic", help="Raw MQTT topic for ad-hoc logging.", type=P),
|
|
32
|
+
),
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
@cli.command("dump")
|
|
36
|
+
@click.option("-l","--one-line",is_flag=True,help="single line per entry")
|
|
37
|
+
@click.pass_obj
|
|
38
|
+
async def dump_(obj,one_line):
|
|
39
|
+
"""Emit a server's (sub)state as a list / YAML file."""
|
|
40
|
+
if not one_line:
|
|
41
|
+
await data_get(obj, obj.server._path, recursive=True)
|
|
42
|
+
return
|
|
43
|
+
for n in obj.server.all_children:
|
|
44
|
+
if n is obj.server:
|
|
45
|
+
continue
|
|
46
|
+
print(n, file=obj.stdout)
|
|
47
|
+
|
|
48
|
+
@cli.group("at", invoke_without_command=True, short_help="create/show/delete an entry")
|
|
49
|
+
@click.argument("path", nargs=1, type=P)
|
|
50
|
+
@click.pass_context
|
|
51
|
+
async def at_cli(ctx, path):
|
|
52
|
+
obj = ctx.obj
|
|
53
|
+
if len(path) == 0 or None in path:
|
|
54
|
+
raise click.UsageError("Path cannot be empty or contain 'None'")
|
|
55
|
+
obj.subpath = path
|
|
56
|
+
obj.node = obj.server.follow(path)
|
|
57
|
+
if ctx.invoked_subcommand is None:
|
|
58
|
+
await data_get(obj, obj.server._path+obj.subpath, recursive=False)
|
|
59
|
+
|
|
60
|
+
@at_cli.command("--help", hidden=True)
|
|
61
|
+
@click.pass_context
|
|
62
|
+
def help(ctx):
|
|
63
|
+
print(at_cli.get_help(ctx))
|
|
64
|
+
|
|
65
|
+
@at_cli.command("dump")
|
|
66
|
+
@click.pass_obj
|
|
67
|
+
@click.option("-l","--one-line",is_flag=True,help="single line per entry")
|
|
68
|
+
async def dump_at(obj,one_line):
|
|
69
|
+
"""Emit a subtree as a list / YAML file."""
|
|
70
|
+
if one_line:
|
|
71
|
+
await data_get(obj, obj.server._path+obj.subpath, recursive=True)
|
|
72
|
+
return
|
|
73
|
+
for n in obj.node.all_children:
|
|
74
|
+
print(n, file=obj.stdout)
|
|
75
|
+
|
|
76
|
+
@at_cli.command("add", short_help="Add an entry")
|
|
77
|
+
@click.option("-f", "--force", is_flag=True, help="Allow replacing an existing entry?")
|
|
78
|
+
@click.option("-m", "--mode", help="DS mode. Default: 'gauge'", default="gauge")
|
|
79
|
+
@click.option(
|
|
80
|
+
"-a",
|
|
81
|
+
"--attr",
|
|
82
|
+
help="The attribute to fetch. Default: the stored value is used directly.",
|
|
83
|
+
default=":",
|
|
84
|
+
type=P,
|
|
85
|
+
)
|
|
86
|
+
@click.argument("source", nargs=1, type=P)
|
|
87
|
+
@click.argument("series", nargs=1)
|
|
88
|
+
@click.argument("tags", nargs=-1)
|
|
89
|
+
@click.pass_obj
|
|
90
|
+
async def add_(obj, source, mode, attr, series, tags, force):
|
|
91
|
+
"""Add a series to Akumuli.
|
|
92
|
+
\b
|
|
93
|
+
path: the name of this copy command. Unique path, non-empty.
|
|
94
|
+
source: the element with the data. unique path, non-empty.
|
|
95
|
+
series: the Akumuli series to write to.
|
|
96
|
+
tags: any number of "name=value" Akumuli tags to use for the series.
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
if not force and obj.node.chain is not None:
|
|
100
|
+
raise click.UsageError("This node already exists. Use '--force' or 'set'.")
|
|
101
|
+
source = P(source)
|
|
102
|
+
mode = getattr(DS, mode)
|
|
103
|
+
tagged = {}
|
|
104
|
+
|
|
105
|
+
if not tags:
|
|
106
|
+
raise click.UsageError("You can't write to a series without tags")
|
|
107
|
+
for x in tags:
|
|
108
|
+
try:
|
|
109
|
+
k, v = x.split("=", 2)
|
|
110
|
+
except ValueError:
|
|
111
|
+
raise click.UsageError("Tags must be key=value") from None
|
|
112
|
+
tagged[k] = v
|
|
113
|
+
|
|
114
|
+
val = dict(source=source, series=series, tags=tagged, mode=mode.name)
|
|
115
|
+
if attr:
|
|
116
|
+
val["attr"] = attr
|
|
117
|
+
try:
|
|
118
|
+
res = (await obj.client.get(source)).value
|
|
119
|
+
if attr:
|
|
120
|
+
res = attrdict._get(res, attr)
|
|
121
|
+
if not isinstance(res,(int,float)):
|
|
122
|
+
raise ValueError(res)
|
|
123
|
+
except (AttributeError,KeyError):
|
|
124
|
+
raise click.UsageError("The value at %s does not exist." % (source,)) from None
|
|
125
|
+
except ValueError:
|
|
126
|
+
raise click.UsageError("The value at %s is not a number." % (source,)) from None
|
|
127
|
+
|
|
128
|
+
res = await obj.client.set(obj.server._path + obj.subpath, val)
|
|
129
|
+
if obj.meta:
|
|
130
|
+
yprint(res, stream=obj.stdout)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@at_cli.command("delete")
|
|
134
|
+
@click.pass_obj
|
|
135
|
+
async def delete_(obj):
|
|
136
|
+
"""Remove a series from Akumuli's copying.
|
|
137
|
+
|
|
138
|
+
\b
|
|
139
|
+
path: the name of the series. Unique path, non-empty.
|
|
140
|
+
|
|
141
|
+
The data set is not physically deleted (with Akumuli that's
|
|
142
|
+
impossible), but no new copying will happen.
|
|
143
|
+
"""
|
|
144
|
+
res = await obj.client.delete(obj.server._path + obj.subpath, nchain=obj.meta or 1)
|
|
145
|
+
if not res.chain:
|
|
146
|
+
raise click.UsageError("This entry doesn't exist.")
|
|
147
|
+
if obj.meta:
|
|
148
|
+
yprint(res, stream=obj.stdout)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
@at_cli.command("set")
|
|
152
|
+
@attr_args
|
|
153
|
+
@click.pass_obj
|
|
154
|
+
async def attr_(obj, vars_, eval_, path_):
|
|
155
|
+
"""Modify a given akumuli series (copier).
|
|
156
|
+
"""
|
|
157
|
+
if not vars_ and not eval_ and not path_:
|
|
158
|
+
return
|
|
159
|
+
|
|
160
|
+
res = await node_attr(obj, obj.server._path + obj.subpath, vars_, eval_, path_)
|
|
161
|
+
if obj.meta:
|
|
162
|
+
yprint(res, stream=obj.stdout)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@cli.command()
|
|
166
|
+
@click.pass_obj
|
|
167
|
+
@click.argument("paths", nargs=-1, type=P)
|
|
168
|
+
async def monitor(obj, paths):
|
|
169
|
+
"""Stand-alone task to monitor a single Akumuli tree"""
|
|
170
|
+
from .task import task
|
|
171
|
+
from .model import AkumuliRoot
|
|
172
|
+
|
|
173
|
+
server = await AkumuliRoot.as_handler(obj.client)
|
|
174
|
+
await server.wait_loaded()
|
|
175
|
+
|
|
176
|
+
async with as_service(obj) as srv:
|
|
177
|
+
await task(obj.client, obj.cfg.kv.akumuli, server[obj.server._name], evt=srv, paths=paths)
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"""
|
|
2
|
+
MoaT-KV client data model for Akumuli
|
|
3
|
+
"""
|
|
4
|
+
import anyio
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
|
|
7
|
+
from moat.util import NotGiven, attrdict, Path
|
|
8
|
+
from moat.kv.obj import ClientEntry, ClientRoot, AttrClientEntry
|
|
9
|
+
from moat.kv.errors import ErrorRoot
|
|
10
|
+
from asyncakumuli import Entry, DS
|
|
11
|
+
|
|
12
|
+
import logging
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _test_hook(e: Entry): # pylint: disable=unused-argument
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class _AkumuliBase(ClientEntry):
|
|
22
|
+
"""
|
|
23
|
+
Forward ``_update_server`` calls to child entries.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
_server = None
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def server(self):
|
|
30
|
+
return self.parent.server
|
|
31
|
+
|
|
32
|
+
async def set_value(self, val): # pylint: disable=arguments-differ
|
|
33
|
+
await super().set_value(val)
|
|
34
|
+
if self.server is not None:
|
|
35
|
+
await self._update_server()
|
|
36
|
+
|
|
37
|
+
async def update_server(self):
|
|
38
|
+
await self.parent.update_server()
|
|
39
|
+
|
|
40
|
+
async def _update_server(self):
|
|
41
|
+
if not self.val_d(True, "present"):
|
|
42
|
+
return
|
|
43
|
+
await self.setup()
|
|
44
|
+
for k in self:
|
|
45
|
+
await k._update_server()
|
|
46
|
+
|
|
47
|
+
async def setup(self):
|
|
48
|
+
pass
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class AkumuliNode(_AkumuliBase, AttrClientEntry):
|
|
52
|
+
"""
|
|
53
|
+
Base class for a node with data (possibly).
|
|
54
|
+
"""
|
|
55
|
+
attr = None
|
|
56
|
+
mode = None
|
|
57
|
+
source = None
|
|
58
|
+
series = None
|
|
59
|
+
factor = 1
|
|
60
|
+
offset = 0
|
|
61
|
+
tags = None
|
|
62
|
+
t_min = None
|
|
63
|
+
ATTRS = ('source', 'attr', 'mode', 'series', 'tags', 't_min', 'factor', 'offset')
|
|
64
|
+
|
|
65
|
+
_work = None
|
|
66
|
+
_t_last = None
|
|
67
|
+
disabled = False
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def tg(self):
|
|
71
|
+
return self.parent.tg
|
|
72
|
+
|
|
73
|
+
def __str__(self):
|
|
74
|
+
return f"N {Path(*self.subpath[1:])} {Path(*self.source)} {Path(*self.attr)} {self.series} {' '.join('%s=%s' % (k,v) for k,v in self.tags.items())}"
|
|
75
|
+
|
|
76
|
+
def _update_disable(self, off):
|
|
77
|
+
self.disabled = off
|
|
78
|
+
for k in self:
|
|
79
|
+
k._update_disable(off)
|
|
80
|
+
|
|
81
|
+
async def with_output(self, evt, src, attr, series, tags, mode):
|
|
82
|
+
"""
|
|
83
|
+
Task that monitors one entry and writes its value to Akumuli.
|
|
84
|
+
"""
|
|
85
|
+
with anyio.CancelScope() as sc:
|
|
86
|
+
self._work = sc
|
|
87
|
+
async with self.client.watch(src, min_depth=0, max_depth=0, fetch=True) as wp:
|
|
88
|
+
evt.set()
|
|
89
|
+
async for msg in wp:
|
|
90
|
+
try:
|
|
91
|
+
val = msg.value
|
|
92
|
+
except AttributeError:
|
|
93
|
+
if msg.get("state", "") != "uptodate":
|
|
94
|
+
await self.root.err.record_error(
|
|
95
|
+
"akumuli",
|
|
96
|
+
self.subpath,
|
|
97
|
+
message="Missing value: {msg}",
|
|
98
|
+
data={"path": self.subpath, "msg": msg},
|
|
99
|
+
)
|
|
100
|
+
continue
|
|
101
|
+
if self.t_min is not None:
|
|
102
|
+
t = anyio.current_time()
|
|
103
|
+
if self._t_last is not None and self._t_last+self.t_min < t:
|
|
104
|
+
continue
|
|
105
|
+
self._t_last = t
|
|
106
|
+
|
|
107
|
+
oval = val
|
|
108
|
+
for k in attr:
|
|
109
|
+
try:
|
|
110
|
+
val = val[k]
|
|
111
|
+
except KeyError:
|
|
112
|
+
await self.root.err.record_error(
|
|
113
|
+
"akumuli",
|
|
114
|
+
self.subpath,
|
|
115
|
+
data=dict(value=oval, attr=attr, message="Missing attr"),
|
|
116
|
+
)
|
|
117
|
+
continue
|
|
118
|
+
|
|
119
|
+
val = val*self.factor+self.offset
|
|
120
|
+
e = Entry(series=series, mode=mode, value=val, tags=tags)
|
|
121
|
+
_test_hook(e)
|
|
122
|
+
await self.server.put(e)
|
|
123
|
+
await self.root.err.record_working("akumuli", self.subpath)
|
|
124
|
+
|
|
125
|
+
async def setup(self):
|
|
126
|
+
await super().setup()
|
|
127
|
+
if self._work is not None:
|
|
128
|
+
await self._work.cancel()
|
|
129
|
+
self._work = None
|
|
130
|
+
if self.server is None:
|
|
131
|
+
return
|
|
132
|
+
|
|
133
|
+
if self.value is NotGiven:
|
|
134
|
+
await self.root.err.record_working("akumuli", self.subpath, comment="deleted")
|
|
135
|
+
return
|
|
136
|
+
data = self.value_or({}, Mapping)
|
|
137
|
+
|
|
138
|
+
src = data.get("source", None)
|
|
139
|
+
series = data.get("series", None)
|
|
140
|
+
tags = data.get("tags", None)
|
|
141
|
+
attr = data.get("attr", ())
|
|
142
|
+
mode = data.get("mode", DS.gauge)
|
|
143
|
+
|
|
144
|
+
if src is None or len(src) == 0 or series is None or not tags or mode is None:
|
|
145
|
+
await self.root.err.record_error(
|
|
146
|
+
"akumuli", self.subpath, data=self.value, message="incomplete data"
|
|
147
|
+
)
|
|
148
|
+
return
|
|
149
|
+
|
|
150
|
+
if self.disabled:
|
|
151
|
+
return
|
|
152
|
+
|
|
153
|
+
if isinstance(mode, str):
|
|
154
|
+
mode = getattr(DS, mode, None)
|
|
155
|
+
|
|
156
|
+
evt = anyio.Event()
|
|
157
|
+
self.tg.start_soon(self.with_output, evt, src, attr, series, tags, mode)
|
|
158
|
+
await evt.wait()
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class AkumuliServer(_AkumuliBase, AttrClientEntry):
|
|
162
|
+
_server = None
|
|
163
|
+
host:str = None
|
|
164
|
+
port:int = None
|
|
165
|
+
|
|
166
|
+
topic:str = None
|
|
167
|
+
|
|
168
|
+
ATTRS=("topic",)
|
|
169
|
+
AUX_ATTRS=("host","port")
|
|
170
|
+
|
|
171
|
+
def __str__(self):
|
|
172
|
+
res = f"{self._name}: {self.host}:{self.port}"
|
|
173
|
+
if self.topic:
|
|
174
|
+
res += " Topic:"+str(self.topic)
|
|
175
|
+
return res
|
|
176
|
+
|
|
177
|
+
@classmethod
|
|
178
|
+
def child_type(cls, name):
|
|
179
|
+
return AkumuliNode
|
|
180
|
+
|
|
181
|
+
@property
|
|
182
|
+
def server(self):
|
|
183
|
+
return self._server
|
|
184
|
+
|
|
185
|
+
@property
|
|
186
|
+
def tg(self):
|
|
187
|
+
return self._server._distkv__tg
|
|
188
|
+
|
|
189
|
+
async def set_value(self, val):
|
|
190
|
+
if val is NotGiven:
|
|
191
|
+
return
|
|
192
|
+
self.host = val.get("server",{}).get("host",None)
|
|
193
|
+
self.port = val.get("server",{}).get("port",None)
|
|
194
|
+
self.topic = val.get("topic",None)
|
|
195
|
+
|
|
196
|
+
def get_value(self, **kw):
|
|
197
|
+
res = super().get_value(**kw)
|
|
198
|
+
try:
|
|
199
|
+
s = res["server"]
|
|
200
|
+
except KeyError:
|
|
201
|
+
res["server"] = s = {}
|
|
202
|
+
s['host'] = self.host
|
|
203
|
+
s['port'] = self.port
|
|
204
|
+
return res
|
|
205
|
+
|
|
206
|
+
async def set_server(self, server):
|
|
207
|
+
self._server = server
|
|
208
|
+
await self._update_server()
|
|
209
|
+
|
|
210
|
+
def set_paths(self, paths):
|
|
211
|
+
"""set enabled paths. Empty: all are on"""
|
|
212
|
+
for v in self:
|
|
213
|
+
v._update_disable(bool(paths))
|
|
214
|
+
for p in paths:
|
|
215
|
+
v = self
|
|
216
|
+
for k in p:
|
|
217
|
+
try:
|
|
218
|
+
v = v[k]
|
|
219
|
+
except KeyError:
|
|
220
|
+
break
|
|
221
|
+
else:
|
|
222
|
+
v.disabled = False
|
|
223
|
+
|
|
224
|
+
async def flush(self):
|
|
225
|
+
await self.server.flush()
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
class AkumuliRoot(_AkumuliBase, ClientRoot):
|
|
229
|
+
CFG = "akumuli"
|
|
230
|
+
err = None
|
|
231
|
+
|
|
232
|
+
async def run_starting(self, server=None): # pylint: disable=arguments-differ
|
|
233
|
+
self._server = server
|
|
234
|
+
if self.err is None:
|
|
235
|
+
self.err = await ErrorRoot.as_handler(self.client)
|
|
236
|
+
await super().run_starting()
|
|
237
|
+
|
|
238
|
+
def child_type(self, name):
|
|
239
|
+
return AkumuliServer
|
|
240
|
+
|
|
241
|
+
async def update_server(self):
|
|
242
|
+
await self._update_server()
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Akumuli task for MoaT-KV
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import anyio
|
|
6
|
+
import asyncakumuli as akumuli
|
|
7
|
+
import socket
|
|
8
|
+
from pprint import pformat
|
|
9
|
+
|
|
10
|
+
try:
|
|
11
|
+
from collections.abc import Mapping
|
|
12
|
+
except ImportError:
|
|
13
|
+
from collections import Mapping
|
|
14
|
+
|
|
15
|
+
from moat.util import combine_dict
|
|
16
|
+
from moat.kv.exceptions import ClientConnectionError
|
|
17
|
+
from .model import AkumuliServer
|
|
18
|
+
|
|
19
|
+
from asyncakumuli import Entry, DS
|
|
20
|
+
|
|
21
|
+
import logging
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
async def task(client, cfg, server: AkumuliServer, paths=(), evt=None): # pylint: disable=unused-argument
|
|
27
|
+
cfg = combine_dict(
|
|
28
|
+
server.value_or({}, Mapping).get("server", {}),
|
|
29
|
+
server.parent.value_or({}, Mapping).get("server", {}),
|
|
30
|
+
cfg["server_default"],
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
async def process_raw(self):
|
|
34
|
+
async with client.msg_monitor(server.topic) as mon:
|
|
35
|
+
async for msg in mon:
|
|
36
|
+
try:
|
|
37
|
+
msg = msg["data"]
|
|
38
|
+
except KeyError:
|
|
39
|
+
continue
|
|
40
|
+
try:
|
|
41
|
+
msg.setdefault("mode", DS.gauge)
|
|
42
|
+
tags = msg.setdefault("tags", {})
|
|
43
|
+
for k,v in tags.items():
|
|
44
|
+
if isinstance(str,bytes):
|
|
45
|
+
tags[k] = v.decode("utf-8")
|
|
46
|
+
else:
|
|
47
|
+
tags[k] = str(v)
|
|
48
|
+
# no-op if it's already a string
|
|
49
|
+
|
|
50
|
+
e = Entry(**msg)
|
|
51
|
+
await srv.put(e)
|
|
52
|
+
except Exception:
|
|
53
|
+
logger.exception("Bad message on %s: \n%s", server.topic, pformat(msg))
|
|
54
|
+
|
|
55
|
+
try:
|
|
56
|
+
async with anyio.create_task_group() as tg:
|
|
57
|
+
async with akumuli.connect(**cfg) as srv:
|
|
58
|
+
srv._distkv__tg = tg
|
|
59
|
+
server.set_paths(paths)
|
|
60
|
+
await server.set_server(srv)
|
|
61
|
+
if evt is not None:
|
|
62
|
+
evt.set()
|
|
63
|
+
|
|
64
|
+
if server.topic is not None:
|
|
65
|
+
await tg.start(process_raw)
|
|
66
|
+
while True:
|
|
67
|
+
await anyio.sleep(99999)
|
|
68
|
+
except TimeoutError:
|
|
69
|
+
raise
|
|
70
|
+
except socket.error as e: # this would eat TimeoutError
|
|
71
|
+
raise ClientConnectionError(cfg["host"], cfg["port"]) from e
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
[Unit]
|
|
2
|
+
Description=MoaT-KV-Akumuli server
|
|
3
|
+
After=moat-kv.service
|
|
4
|
+
Requires=moat-kv.service
|
|
5
|
+
|
|
6
|
+
ConditionFileNotEmpty=/etc/moat/moat.cfg
|
|
7
|
+
|
|
8
|
+
[Install]
|
|
9
|
+
WantedBy=multi-user.target
|
|
10
|
+
|
|
11
|
+
[Service]
|
|
12
|
+
Type=notify
|
|
13
|
+
ExecStart=/usr/bin/moat kv akumuli %I monitor
|
|
14
|
+
|
|
15
|
+
EnvironmentFile=/usr/lib/moat/kv/env
|
|
16
|
+
EnvironmentFile=-/etc/moat/kv.env
|
|
17
|
+
|
|
18
|
+
TimeoutSec=300
|
|
19
|
+
WatchdogSec=10
|
|
20
|
+
|
|
21
|
+
Restart=always
|
|
22
|
+
RestartSec=30
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: moat-kv-akumuli
|
|
3
|
+
Version: 0.8.3
|
|
4
|
+
Summary: Akumuli storage connector for MoaT-KV
|
|
5
|
+
Author-email: Matthias Urlichs <matthias@urlichs.de>
|
|
6
|
+
Project-URL: homepage, https://m-o-a-t.org
|
|
7
|
+
Project-URL: repository, https://github.com/M-o-a-T/moat-kv-akumuli
|
|
8
|
+
Keywords: MoaT
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Information Technology
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Database
|
|
15
|
+
Classifier: Topic :: Home Automation
|
|
16
|
+
Requires-Python: >=3.8
|
|
17
|
+
Description-Content-Type: text/x-rst
|
|
18
|
+
|
|
19
|
+
===========
|
|
20
|
+
DistAkumuli
|
|
21
|
+
===========
|
|
22
|
+
|
|
23
|
+
DistAkumuli is a simple link between Akumuli and DistKV.
|
|
24
|
+
|
|
25
|
+
It will update an Akumuli series whenever a DistKV value changes.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
.gitignore
|
|
2
|
+
.pylintrc
|
|
3
|
+
Makefile
|
|
4
|
+
README.rst
|
|
5
|
+
TODO.rst
|
|
6
|
+
USAGE.rst
|
|
7
|
+
moat-kv-akumuli@.service
|
|
8
|
+
pyproject.toml
|
|
9
|
+
moat/kv/akumuli/__init__.py
|
|
10
|
+
moat/kv/akumuli/_config.yaml
|
|
11
|
+
moat/kv/akumuli/_main.py
|
|
12
|
+
moat/kv/akumuli/model.py
|
|
13
|
+
moat/kv/akumuli/task.py
|
|
14
|
+
moat_kv_akumuli.egg-info/PKG-INFO
|
|
15
|
+
moat_kv_akumuli.egg-info/SOURCES.txt
|
|
16
|
+
moat_kv_akumuli.egg-info/dependency_links.txt
|
|
17
|
+
moat_kv_akumuli.egg-info/requires.txt
|
|
18
|
+
moat_kv_akumuli.egg-info/top_level.txt
|
|
19
|
+
tests/__init__.py
|
|
20
|
+
tests/conftest.py
|
|
21
|
+
tests/test_akumuli.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
moat-kv~=0.70.2
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
moat
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
build-backend = "setuptools.build_meta"
|
|
3
|
+
requires = [ "setuptools", "wheel", "setuptools-scm",]
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
classifiers = [
|
|
7
|
+
"Development Status :: 4 - Beta",
|
|
8
|
+
"Intended Audience :: Information Technology",
|
|
9
|
+
"License :: OSI Approved :: MIT License",
|
|
10
|
+
"License :: OSI Approved :: Apache Software License",
|
|
11
|
+
"Programming Language :: Python :: 3",
|
|
12
|
+
"Topic :: Database",
|
|
13
|
+
"Topic :: Home Automation",
|
|
14
|
+
]
|
|
15
|
+
dependencies = [
|
|
16
|
+
"moat-kv ~= 0.70.2",
|
|
17
|
+
]
|
|
18
|
+
dynamic = [ "version",]
|
|
19
|
+
keywords = [ "MoaT",]
|
|
20
|
+
requires-python = ">=3.8"
|
|
21
|
+
name = "moat-kv-akumuli"
|
|
22
|
+
description = "Akumuli storage connector for MoaT-KV"
|
|
23
|
+
readme = "README.rst"
|
|
24
|
+
[[project.authors]]
|
|
25
|
+
email = "matthias@urlichs.de"
|
|
26
|
+
name = "Matthias Urlichs"
|
|
27
|
+
|
|
28
|
+
[project.license]
|
|
29
|
+
file = "LICENSE"
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
homepage = "https://m-o-a-t.org"
|
|
33
|
+
repository = "https://github.com/M-o-a-T/moat-kv-akumuli"
|
|
34
|
+
|
|
35
|
+
[tool.flake8]
|
|
36
|
+
max-line-length = 99
|
|
37
|
+
ignore = [ "F841", "F401", "E731", "E502", "E402", "E127", "E123", "W503", "E231", "E203", "E501" ]
|
|
38
|
+
|
|
39
|
+
[tool.isort]
|
|
40
|
+
line_length = 99
|
|
41
|
+
multi_line_output = 3
|
|
42
|
+
profile = "black"
|
|
43
|
+
|
|
44
|
+
[tool.setuptools]
|
|
45
|
+
packages = [ "moat.kv.akumuli",]
|
|
46
|
+
[tool.setuptools.package-data]
|
|
47
|
+
"*" = ["*.yaml"]
|
|
48
|
+
|
|
49
|
+
[tool.setuptools_scm]
|
|
50
|
+
|
|
51
|
+
[tool.black]
|
|
52
|
+
line-length = 99
|
|
53
|
+
|
|
54
|
+
[tool.tox]
|
|
55
|
+
legacy_tox_ini = """
|
|
56
|
+
[tox]
|
|
57
|
+
isolated_build = True
|
|
58
|
+
envlist = py310,check
|
|
59
|
+
|
|
60
|
+
[testenv]
|
|
61
|
+
setenv =
|
|
62
|
+
PYTHONPATH = {env:PYTHONPATH}{:}{toxinidir}
|
|
63
|
+
deps =
|
|
64
|
+
anyio
|
|
65
|
+
asyncwebsockets
|
|
66
|
+
asyncclick
|
|
67
|
+
asyncscope
|
|
68
|
+
trio
|
|
69
|
+
pytest
|
|
70
|
+
commands =
|
|
71
|
+
python3 -mpytest tests/
|
|
72
|
+
|
|
73
|
+
[testenv:check]
|
|
74
|
+
commands =
|
|
75
|
+
pylint moat tests
|
|
76
|
+
flake8p moat tests
|
|
77
|
+
black --check moat tests
|
|
78
|
+
deps =
|
|
79
|
+
pytest
|
|
80
|
+
pylint
|
|
81
|
+
black
|
|
82
|
+
flake8-pyproject
|
|
83
|
+
flake8
|
|
84
|
+
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
[tool.pytest]
|
|
88
|
+
filterwarnings = [
|
|
89
|
+
"error",
|
|
90
|
+
"ignore:unclosed:ResourceWarning",
|
|
91
|
+
]
|
|
92
|
+
addopts = "--verbose"
|
|
93
|
+
|
|
94
|
+
[tool.pylint]
|
|
95
|
+
[tool.pylint.messages_control]
|
|
96
|
+
disable = "wrong-import-order,ungrouped-imports,too-many-nested-blocks,use-dict-literal,unspecified-encoding,too-many-statements,too-many-return-statements,too-many-locals,too-many-instance-attributes,too-many-branches,too-many-arguments,too-few-public-methods,superfluous-parens,no-else-return,no-else-continue,invalid-name,fixme"
|
|
97
|
+
|
|
98
|
+
[tool.moat]
|
|
File without changes
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import anyio
|
|
2
|
+
from time import time
|
|
3
|
+
|
|
4
|
+
from moat.util import P, load_ext
|
|
5
|
+
from moat.kv.mock.mqtt import stdtest
|
|
6
|
+
|
|
7
|
+
from asyncakumuli.mock import Tester, TCP_PORT
|
|
8
|
+
|
|
9
|
+
task = akumuli_task = load_ext("moat.kv.akumuli.task", "task", err=True)
|
|
10
|
+
|
|
11
|
+
akumuli_model = load_ext("moat.kv.akumuli.model", err=True)
|
|
12
|
+
AkumuliRoot = akumuli_model.AkumuliRoot
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _hook(e):
|
|
16
|
+
e.ns_time = int(time() * 1000000000)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
akumuli_model._test_hook = _hook
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
async def test_basic(): # no autojump
|
|
23
|
+
async with (
|
|
24
|
+
stdtest(test_0={"init": 125}, n=1, tocks=200) as st,
|
|
25
|
+
st.client(0) as client,
|
|
26
|
+
Tester().run() as t,
|
|
27
|
+
):
|
|
28
|
+
await st.run(f"akumuli test add -h 127.0.0.1 -p {TCP_PORT}")
|
|
29
|
+
await client.set(P("test.one.two"), value=41)
|
|
30
|
+
await st.run("akumuli test at test.foo.bar add test.one.two whatever foo=bar")
|
|
31
|
+
aki = await AkumuliRoot.as_handler(client)
|
|
32
|
+
aki._cfg.server_default.port = TCP_PORT
|
|
33
|
+
st.tg.start_soon(task, client, aki._cfg, aki["test"])
|
|
34
|
+
await anyio.sleep(1)
|
|
35
|
+
await aki["test"].flush()
|
|
36
|
+
await client.set(P("test.one.two"), value=42)
|
|
37
|
+
await anyio.sleep(0.3)
|
|
38
|
+
await aki["test"].flush()
|
|
39
|
+
await anyio.sleep(0.8)
|
|
40
|
+
await client.set(P("test.one.two"), value=43)
|
|
41
|
+
await anyio.sleep(0.3)
|
|
42
|
+
await aki["test"].flush()
|
|
43
|
+
|
|
44
|
+
n = 0
|
|
45
|
+
async for x in t.get_data("whatever", tags={}, t_start=time() - 1000, t_end=time() + 1000):
|
|
46
|
+
n += 1
|
|
47
|
+
assert x.value in (41,42,43)
|
|
48
|
+
assert abs(time() - x.time) < 10
|
|
49
|
+
assert n > 1
|
|
50
|
+
pass
|