hat-event 0.9.27__cp310.cp311.cp312.cp313-abi3-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hat/event/__init__.py +1 -0
- hat/event/adminer/__init__.py +18 -0
- hat/event/adminer/client.py +124 -0
- hat/event/adminer/common.py +27 -0
- hat/event/adminer/server.py +111 -0
- hat/event/backends/__init__.py +0 -0
- hat/event/backends/dummy.py +49 -0
- hat/event/backends/lmdb/__init__.py +9 -0
- hat/event/backends/lmdb/backend.py +319 -0
- hat/event/backends/lmdb/common.py +277 -0
- hat/event/backends/lmdb/conditions.py +102 -0
- hat/event/backends/lmdb/convert/__init__.py +0 -0
- hat/event/backends/lmdb/convert/__main__.py +8 -0
- hat/event/backends/lmdb/convert/convert_v06_to_v07.py +213 -0
- hat/event/backends/lmdb/convert/convert_v07_to_v09.py +175 -0
- hat/event/backends/lmdb/convert/main.py +88 -0
- hat/event/backends/lmdb/convert/v06.py +216 -0
- hat/event/backends/lmdb/convert/v07.py +508 -0
- hat/event/backends/lmdb/convert/v09.py +50 -0
- hat/event/backends/lmdb/convert/version.py +63 -0
- hat/event/backends/lmdb/environment.py +100 -0
- hat/event/backends/lmdb/latestdb.py +116 -0
- hat/event/backends/lmdb/manager/__init__.py +0 -0
- hat/event/backends/lmdb/manager/__main__.py +8 -0
- hat/event/backends/lmdb/manager/common.py +45 -0
- hat/event/backends/lmdb/manager/copy.py +92 -0
- hat/event/backends/lmdb/manager/main.py +34 -0
- hat/event/backends/lmdb/manager/query.py +215 -0
- hat/event/backends/lmdb/refdb.py +234 -0
- hat/event/backends/lmdb/systemdb.py +102 -0
- hat/event/backends/lmdb/timeseriesdb.py +486 -0
- hat/event/backends/memory.py +178 -0
- hat/event/common/__init__.py +144 -0
- hat/event/common/backend.py +91 -0
- hat/event/common/collection/__init__.py +8 -0
- hat/event/common/collection/common.py +28 -0
- hat/event/common/collection/list.py +19 -0
- hat/event/common/collection/tree.py +62 -0
- hat/event/common/common.py +176 -0
- hat/event/common/encoder.py +305 -0
- hat/event/common/json_schema_repo.json +1 -0
- hat/event/common/matches.py +44 -0
- hat/event/common/module.py +142 -0
- hat/event/common/sbs_repo.json +1 -0
- hat/event/common/subscription/__init__.py +22 -0
- hat/event/common/subscription/_csubscription.abi3.pyd +0 -0
- hat/event/common/subscription/common.py +145 -0
- hat/event/common/subscription/csubscription.py +47 -0
- hat/event/common/subscription/pysubscription.py +97 -0
- hat/event/component.py +284 -0
- hat/event/eventer/__init__.py +28 -0
- hat/event/eventer/client.py +260 -0
- hat/event/eventer/common.py +27 -0
- hat/event/eventer/server.py +286 -0
- hat/event/manager/__init__.py +0 -0
- hat/event/manager/__main__.py +8 -0
- hat/event/manager/common.py +48 -0
- hat/event/manager/main.py +387 -0
- hat/event/server/__init__.py +0 -0
- hat/event/server/__main__.py +8 -0
- hat/event/server/adminer_server.py +43 -0
- hat/event/server/engine.py +216 -0
- hat/event/server/engine_runner.py +127 -0
- hat/event/server/eventer_client.py +205 -0
- hat/event/server/eventer_client_runner.py +152 -0
- hat/event/server/eventer_server.py +119 -0
- hat/event/server/main.py +84 -0
- hat/event/server/main_runner.py +212 -0
- hat_event-0.9.27.dist-info/LICENSE +202 -0
- hat_event-0.9.27.dist-info/METADATA +108 -0
- hat_event-0.9.27.dist-info/RECORD +73 -0
- hat_event-0.9.27.dist-info/WHEEL +7 -0
- hat_event-0.9.27.dist-info/entry_points.txt +5 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import struct
|
|
3
|
+
|
|
4
|
+
from hat import json
|
|
5
|
+
from hat import sbs
|
|
6
|
+
from hat import util
|
|
7
|
+
|
|
8
|
+
from hat.event.common.common import (Event,
|
|
9
|
+
EventId,
|
|
10
|
+
EventPayload,
|
|
11
|
+
EventPayloadBinary,
|
|
12
|
+
EventPayloadJson,
|
|
13
|
+
Order,
|
|
14
|
+
OrderBy,
|
|
15
|
+
QueryLatestParams,
|
|
16
|
+
QueryParams,
|
|
17
|
+
QueryResult,
|
|
18
|
+
QueryServerParams,
|
|
19
|
+
QueryTimeseriesParams,
|
|
20
|
+
RegisterEvent,
|
|
21
|
+
Status,
|
|
22
|
+
Timestamp)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def timestamp_to_bytes(t: Timestamp) -> util.Bytes:
|
|
26
|
+
"""Convert timestamp to 12 byte representation
|
|
27
|
+
|
|
28
|
+
Bytes [0, 8] are big endian unsigned `Timestamp.s` + 2^63 and
|
|
29
|
+
bytes [9, 12] are big endian unsigned `Timestamp.us`.
|
|
30
|
+
|
|
31
|
+
"""
|
|
32
|
+
return struct.pack(">QI", t.s + (1 << 63), t.us)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def timestamp_from_bytes(data: util.Bytes) -> Timestamp:
|
|
36
|
+
"""Create new timestamp from 12 byte representation
|
|
37
|
+
|
|
38
|
+
Bytes representation is same as defined for `timestamp_to_bytes` function.
|
|
39
|
+
|
|
40
|
+
"""
|
|
41
|
+
s, us = struct.unpack(">QI", data)
|
|
42
|
+
return Timestamp(s - (1 << 63), us)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def timestamp_to_float(t: Timestamp) -> float:
|
|
46
|
+
"""Convert timestamp to floating number of seconds since 1970-01-01 UTC
|
|
47
|
+
|
|
48
|
+
For precise serialization see `timestamp_to_bytes`/`timestamp_from_bytes`.
|
|
49
|
+
|
|
50
|
+
"""
|
|
51
|
+
return t.s + t.us * 1E-6
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def timestamp_from_float(ts: float) -> Timestamp:
|
|
55
|
+
"""Create timestamp from floating number of seconds since 1970-01-01 UTC
|
|
56
|
+
|
|
57
|
+
For precise serialization see `timestamp_to_bytes`/`timestamp_from_bytes`.
|
|
58
|
+
|
|
59
|
+
"""
|
|
60
|
+
s = int(ts)
|
|
61
|
+
if ts < 0:
|
|
62
|
+
s = s - 1
|
|
63
|
+
|
|
64
|
+
us = round((ts - s) * 1E6)
|
|
65
|
+
|
|
66
|
+
if us == 1_000_000:
|
|
67
|
+
return Timestamp(s + 1, 0)
|
|
68
|
+
|
|
69
|
+
else:
|
|
70
|
+
return Timestamp(s, us)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def timestamp_to_datetime(t: Timestamp) -> datetime.datetime:
|
|
74
|
+
"""Convert timestamp to datetime (representing utc time)
|
|
75
|
+
|
|
76
|
+
For precise serialization see `timestamp_to_bytes`/`timestamp_from_bytes`.
|
|
77
|
+
|
|
78
|
+
"""
|
|
79
|
+
try:
|
|
80
|
+
dt_from_s = datetime.datetime.fromtimestamp(t.s, datetime.timezone.utc)
|
|
81
|
+
|
|
82
|
+
except OSError:
|
|
83
|
+
dt_from_s = (
|
|
84
|
+
datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) +
|
|
85
|
+
datetime.timedelta(seconds=t.s))
|
|
86
|
+
|
|
87
|
+
return datetime.datetime(
|
|
88
|
+
year=dt_from_s.year,
|
|
89
|
+
month=dt_from_s.month,
|
|
90
|
+
day=dt_from_s.day,
|
|
91
|
+
hour=dt_from_s.hour,
|
|
92
|
+
minute=dt_from_s.minute,
|
|
93
|
+
second=dt_from_s.second,
|
|
94
|
+
microsecond=t.us,
|
|
95
|
+
tzinfo=datetime.timezone.utc)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def timestamp_from_datetime(dt: datetime.datetime) -> Timestamp:
|
|
99
|
+
"""Create new timestamp from datetime
|
|
100
|
+
|
|
101
|
+
If `tzinfo` is not set, it is assumed that provided datetime represents
|
|
102
|
+
utc time.
|
|
103
|
+
|
|
104
|
+
For precise serialization see `timestamp_to_bytes`/`timestamp_from_bytes`.
|
|
105
|
+
|
|
106
|
+
"""
|
|
107
|
+
if not dt.tzinfo:
|
|
108
|
+
dt = dt.replace(tzinfo=datetime.timezone.utc)
|
|
109
|
+
|
|
110
|
+
s = int(dt.timestamp())
|
|
111
|
+
|
|
112
|
+
if dt.timestamp() < 0:
|
|
113
|
+
s = s - 1
|
|
114
|
+
|
|
115
|
+
return Timestamp(s=s, us=dt.microsecond)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def timestamp_to_sbs(t: Timestamp) -> sbs.Data:
|
|
119
|
+
"""Convert timestamp to SBS data"""
|
|
120
|
+
return {'s': t.s, 'us': t.us}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def timestamp_from_sbs(data: sbs.Data) -> Timestamp:
|
|
124
|
+
"""Create new timestamp from SBS data"""
|
|
125
|
+
return Timestamp(s=data['s'], us=data['us'])
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def status_to_sbs(status: Status) -> sbs.Data:
|
|
129
|
+
"""Convert Status to SBS data"""
|
|
130
|
+
return status.value, None
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def status_from_sbs(status: sbs.Data) -> Status:
|
|
134
|
+
"""Create Status based on SBS data"""
|
|
135
|
+
return Status(status[0])
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def event_to_sbs(event: Event) -> sbs.Data:
|
|
139
|
+
"""Convert Event to SBS data"""
|
|
140
|
+
return {'id': _event_id_to_sbs(event.id),
|
|
141
|
+
'type': list(event.type),
|
|
142
|
+
'timestamp': timestamp_to_sbs(event.timestamp),
|
|
143
|
+
'sourceTimestamp': _optional_to_sbs(event.source_timestamp,
|
|
144
|
+
timestamp_to_sbs),
|
|
145
|
+
'payload': _optional_to_sbs(event.payload, event_payload_to_sbs)}
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def event_from_sbs(data: sbs.Data) -> Event:
|
|
149
|
+
"""Create Event based on SBS data"""
|
|
150
|
+
return Event(id=_event_id_from_sbs(data['id']),
|
|
151
|
+
type=tuple(data['type']),
|
|
152
|
+
timestamp=timestamp_from_sbs(data['timestamp']),
|
|
153
|
+
source_timestamp=_optional_from_sbs(data['sourceTimestamp'],
|
|
154
|
+
timestamp_from_sbs),
|
|
155
|
+
payload=_optional_from_sbs(data['payload'],
|
|
156
|
+
event_payload_from_sbs))
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def register_event_to_sbs(event: RegisterEvent) -> sbs.Data:
|
|
160
|
+
"""Convert RegisterEvent to SBS data"""
|
|
161
|
+
return {'type': list(event.type),
|
|
162
|
+
'sourceTimestamp': _optional_to_sbs(event.source_timestamp,
|
|
163
|
+
timestamp_to_sbs),
|
|
164
|
+
'payload': _optional_to_sbs(event.payload, event_payload_to_sbs)}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def register_event_from_sbs(data: sbs.Data) -> RegisterEvent:
|
|
168
|
+
"""Create RegisterEvent based on SBS data"""
|
|
169
|
+
return RegisterEvent(
|
|
170
|
+
type=tuple(data['type']),
|
|
171
|
+
source_timestamp=_optional_from_sbs(data['sourceTimestamp'],
|
|
172
|
+
timestamp_from_sbs),
|
|
173
|
+
payload=_optional_from_sbs(data['payload'], event_payload_from_sbs))
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def query_params_to_sbs(params: QueryParams) -> sbs.Data:
|
|
177
|
+
"""Convert QueryParams to SBS data"""
|
|
178
|
+
if isinstance(params, QueryLatestParams):
|
|
179
|
+
return 'latest', {
|
|
180
|
+
'eventTypes': _optional_to_sbs(params.event_types,
|
|
181
|
+
_event_types_to_sbs)}
|
|
182
|
+
|
|
183
|
+
if isinstance(params, QueryTimeseriesParams):
|
|
184
|
+
return 'timeseries', {
|
|
185
|
+
'eventTypes': _optional_to_sbs(params.event_types,
|
|
186
|
+
_event_types_to_sbs),
|
|
187
|
+
'tFrom': _optional_to_sbs(params.t_from, timestamp_to_sbs),
|
|
188
|
+
'tTo': _optional_to_sbs(params.t_to, timestamp_to_sbs),
|
|
189
|
+
'sourceTFrom': _optional_to_sbs(params.source_t_from,
|
|
190
|
+
timestamp_to_sbs),
|
|
191
|
+
'sourceTTo': _optional_to_sbs(params.source_t_to,
|
|
192
|
+
timestamp_to_sbs),
|
|
193
|
+
'order': (params.order.value, None),
|
|
194
|
+
'orderBy': (params.order_by.value, None),
|
|
195
|
+
'maxResults': _optional_to_sbs(params.max_results),
|
|
196
|
+
'lastEventId': _optional_to_sbs(params.last_event_id,
|
|
197
|
+
_event_id_to_sbs)}
|
|
198
|
+
|
|
199
|
+
if isinstance(params, QueryServerParams):
|
|
200
|
+
return 'server', {
|
|
201
|
+
'serverId': params.server_id,
|
|
202
|
+
'persisted': params.persisted,
|
|
203
|
+
'maxResults': _optional_to_sbs(params.max_results),
|
|
204
|
+
'lastEventId': _optional_to_sbs(params.last_event_id,
|
|
205
|
+
_event_id_to_sbs)}
|
|
206
|
+
|
|
207
|
+
raise ValueError('unsupported params type')
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def query_params_from_sbs(data: sbs.Data) -> QueryParams:
|
|
211
|
+
"""Create QueryParams based on SBS data"""
|
|
212
|
+
if data[0] == 'latest':
|
|
213
|
+
return QueryLatestParams(
|
|
214
|
+
event_types=_optional_from_sbs(data[1]['eventTypes'],
|
|
215
|
+
_event_types_from_sbs))
|
|
216
|
+
|
|
217
|
+
if data[0] == 'timeseries':
|
|
218
|
+
return QueryTimeseriesParams(
|
|
219
|
+
event_types=_optional_from_sbs(data[1]['eventTypes'],
|
|
220
|
+
_event_types_from_sbs),
|
|
221
|
+
t_from=_optional_from_sbs(data[1]['tFrom'], timestamp_from_sbs),
|
|
222
|
+
t_to=_optional_from_sbs(data[1]['tTo'], timestamp_from_sbs),
|
|
223
|
+
source_t_from=_optional_from_sbs(data[1]['sourceTFrom'],
|
|
224
|
+
timestamp_from_sbs),
|
|
225
|
+
source_t_to=_optional_from_sbs(data[1]['sourceTTo'],
|
|
226
|
+
timestamp_from_sbs),
|
|
227
|
+
order=Order(data[1]['order'][0]),
|
|
228
|
+
order_by=OrderBy(data[1]['orderBy'][0]),
|
|
229
|
+
max_results=_optional_from_sbs(data[1]['maxResults']),
|
|
230
|
+
last_event_id=_optional_from_sbs(data[1]['lastEventId'],
|
|
231
|
+
_event_id_from_sbs))
|
|
232
|
+
|
|
233
|
+
if data[0] == 'server':
|
|
234
|
+
return QueryServerParams(
|
|
235
|
+
server_id=data[1]['serverId'],
|
|
236
|
+
persisted=data[1]['persisted'],
|
|
237
|
+
max_results=_optional_from_sbs(data[1]['maxResults']),
|
|
238
|
+
last_event_id=_optional_from_sbs(data[1]['lastEventId'],
|
|
239
|
+
_event_id_from_sbs))
|
|
240
|
+
|
|
241
|
+
raise ValueError('unsupported params type')
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def query_result_to_sbs(result: QueryResult) -> sbs.Data:
|
|
245
|
+
"""Convert QueryResult to SBS data"""
|
|
246
|
+
return {'events': [event_to_sbs(event) for event in result.events],
|
|
247
|
+
'moreFollows': result.more_follows}
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def query_result_from_sbs(data: sbs.Data) -> QueryResult:
|
|
251
|
+
"""Create QueryResult based on SBS data"""
|
|
252
|
+
return QueryResult(events=[event_from_sbs(i) for i in data['events']],
|
|
253
|
+
more_follows=data['moreFollows'])
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def event_payload_to_sbs(payload: EventPayload) -> sbs.Data:
|
|
257
|
+
"""Convert EventPayload to SBS data"""
|
|
258
|
+
if isinstance(payload, EventPayloadBinary):
|
|
259
|
+
return 'binary', {'type': payload.type,
|
|
260
|
+
'data': payload.data}
|
|
261
|
+
|
|
262
|
+
if isinstance(payload, EventPayloadJson):
|
|
263
|
+
return 'json', json.encode(payload.data)
|
|
264
|
+
|
|
265
|
+
raise ValueError('unsupported payload type')
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def event_payload_from_sbs(data: sbs.Data) -> EventPayload:
|
|
269
|
+
"""Create EventPayload based on SBS data"""
|
|
270
|
+
if data[0] == 'binary':
|
|
271
|
+
return EventPayloadBinary(type=data[1]['type'],
|
|
272
|
+
data=data[1]['data'])
|
|
273
|
+
|
|
274
|
+
if data[0] == 'json':
|
|
275
|
+
return EventPayloadJson(data=json.decode(data[1]))
|
|
276
|
+
|
|
277
|
+
raise ValueError('unsupported payload type')
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _event_id_to_sbs(event_id):
|
|
281
|
+
return {'server': event_id.server,
|
|
282
|
+
'session': event_id.session,
|
|
283
|
+
'instance': event_id.instance}
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _event_id_from_sbs(data):
|
|
287
|
+
return EventId(server=data['server'],
|
|
288
|
+
session=data['session'],
|
|
289
|
+
instance=data['instance'])
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _event_types_to_sbs(event_types):
|
|
293
|
+
return [list(event_type) for event_type in event_types]
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _event_types_from_sbs(data):
|
|
297
|
+
return [tuple(i) for i in data]
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _optional_to_sbs(value, fn=lambda i: i):
|
|
301
|
+
return ('value', fn(value)) if value is not None else ('none', None)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _optional_from_sbs(data, fn=lambda i: i):
|
|
305
|
+
return fn(data[1]) if data[0] == 'value' else None
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"hat-event": {"monitor_data.yaml": {"$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "hat-event://monitor_data.yaml", "title": "monitor data", "description": "data property of monitor component info", "type": "object", "required": ["server_id", "eventer_server", "server_token"], "properties": {"server_id": {"type": "integer"}, "eventer_server": {"type": "object", "required": ["host", "port"], "properties": {"host": {"type": "string"}, "port": {"type": "integer"}}}, "server_token": {"type": ["string", "null"]}}}, "events.yaml": {"$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "hat-event://events.yaml", "definitions": {"events": {"engine": {"enum": ["STARTED", "STOPPED"]}, "eventer": {"enum": ["CONNECTED", "DISCONNECTED"]}, "synced": {"oneOf": [{"type": "object", "required": ["state"], "properties": {"state": {"const": "CONNECTED"}}}, {"type": "object", "required": ["state"], "properties": {"state": {"const": "SYNCING"}}}, {"type": "object", "required": ["state", "count"], "properties": {"state": {"const": "SYNCED"}, "count": {"type": ["integer", "null"]}}}]}}}}, "server.yaml": {"$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "hat-event://server.yaml", "title": "Event Server", "description": "Event Server's configuration", "type": "object", "required": ["name", "server_id", "backend", "modules", "eventer_server"], "properties": {"type": {"const": "event", "description": "configuration type identification"}, "version": {"type": "string", "description": "component version"}, "log": {"$ref": "hat-json://logging.yaml"}, "name": {"type": "string", "description": "component name"}, "server_id": {"type": "integer", "description": "server identifier"}, "server_token": {"type": ["string", "null"], "description": "server token"}, "backend": {"$ref": "hat-event://server.yaml#/$defs/backend"}, "modules": {"type": "array", "items": {"$ref": "hat-event://server.yaml#/$defs/module"}}, "eventer_server": {"type": "object", "required": ["host", "port"], "properties": {"host": {"type": "string", "default": "127.0.0.1"}, "port": {"type": "integer", "default": 23012}}}, "adminer_server": {"type": "object", "required": ["host", "port"], "properties": {"host": {"type": "string", "default": "127.0.0.1"}, "port": {"type": "integer", "default": 23015}}}, "monitor_component": {"type": "object", "required": ["host", "port", "group"], "properties": {"host": {"type": "string", "default": "127.0.0.1"}, "port": {"type": "integer", "default": 23010}, "group": {"type": "string"}}}}, "$defs": {"backend": {"type": "object", "description": "structure of backend configuration depends on backend type\n", "required": ["module"], "properties": {"module": {"type": "string", "description": "full python module name that implements backend"}}}, "module": {"type": "object", "description": "structure of module configuration depends on module type\n", "required": ["module"], "properties": {"module": {"type": "string", "description": "full python module name that implements module"}}}}}, "backends/lmdb.yaml": {"$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "hat-event://backends/lmdb.yaml", "title": "LMDB backend", "type": "object", "required": ["db_path", "identifier", "flush_period", "cleanup_period", "conditions", "latest", "timeseries"], "properties": {"db_path": {"type": "string"}, "identifier": {"type": ["null", "string"]}, "flush_period": {"type": "number"}, "cleanup_period": {"type": "number"}, "conditions": {"type": "array", "items": {"type": "object", "required": ["subscriptions", "condition"], "properties": {"subscriptions": {"$ref": "hat-event://backends/lmdb.yaml#/$defs/event_types"}, "condition": {"$ref": "hat-event://backends/lmdb.yaml#/$defs/condition"}}}}, "latest": {"type": "object", "required": ["subscriptions"], "properties": {"subscriptions": {"$ref": "hat-event://backends/lmdb.yaml#/$defs/event_types"}}}, "timeseries": {"type": "array", "items": {"type": "object", "required": ["order_by", "subscriptions"], "properties": {"order_by": {"enum": ["TIMESTAMP", "SOURCE_TIMESTAMP"]}, "subscriptions": {"$ref": "hat-event://backends/lmdb.yaml#/$defs/event_types"}, "limit": {"$ref": "hat-event://backends/lmdb.yaml#/$defs/limit"}}}}}, "$defs": {"event_types": {"type": "array", "items": {"type": "array", "items": {"type": "string"}}}, "limit": {"type": "object", "properties": {"min_entries": {"type": "number", "description": "number of entries kept despite of other limits\n"}, "max_entries": {"type": "number", "description": "maximum number of entries\n"}, "duration": {"type": "number", "description": "limit for the persisted history based on keys\nexpressed as duration in seconds\n"}, "size": {"type": "number", "description": "memory consumption size in bytes that triggers\nadditional cleanup based on average entry size\n"}}}, "condition": {"oneOf": [{"$ref": "hat-event://backends/lmdb.yaml#/$defs/conditions/all"}, {"$ref": "hat-event://backends/lmdb.yaml#/$defs/conditions/any"}, {"$ref": "hat-event://backends/lmdb.yaml#/$defs/conditions/json"}]}, "conditions": {"all": {"type": "object", "required": ["type", "conditions"], "properties": {"type": {"const": "all"}, "conditions": {"type": "array", "items": {"$ref": "hat-event://backends/lmdb.yaml#/$defs/condition"}}}}, "any": {"type": "object", "required": ["type", "conditions"], "properties": {"type": {"const": "any"}, "conditions": {"type": "array", "items": {"$ref": "hat-event://backends/lmdb.yaml#/$defs/condition"}}}}, "json": {"type": "object", "required": ["type"], "properties": {"type": {"const": "json"}, "data_path": {"$ref": "hat-event://backends/lmdb.yaml#/$defs/path"}, "data_type": {"enum": ["null", "boolean", "string", "number", "array", "object"]}, "data_value": {}}}}, "path": {"oneOf": [{"type": "string"}, {"type": "integer"}, {"type": "array", "items": {"$ref": "hat-event://backends/lmdb.yaml#/$defs/path"}}]}}}, "backends/sqlite.yaml": {"$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "hat-event://backends/sqlite.yaml", "title": "Sqlite backend", "description": "Sqlite backend configuration", "type": "object", "required": ["db_path", "query_pool_size"], "properties": {"db_path": {"type": "string", "description": "path to sqlite database file"}, "query_pool_size": {"type": "integer", "description": "number of connections in a pool used for querying\n"}}}}}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from hat.event.common.common import EventType
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def matches_query_type(event_type: EventType,
|
|
5
|
+
query_type: EventType
|
|
6
|
+
) -> bool:
|
|
7
|
+
"""Determine if event type matches query type
|
|
8
|
+
|
|
9
|
+
Event type is tested if it matches query type according to the following
|
|
10
|
+
rules:
|
|
11
|
+
|
|
12
|
+
* Matching is performed on subtypes in increasing order.
|
|
13
|
+
* Event type is a match only if all its subtypes are matched by
|
|
14
|
+
corresponding query subtypes.
|
|
15
|
+
* Matching is finished when all query subtypes are exhausted.
|
|
16
|
+
* Query subtype '?' matches exactly one event subtype of any value.
|
|
17
|
+
The subtype must exist.
|
|
18
|
+
* Query subtype '*' matches 0 or more event subtypes of any value. It
|
|
19
|
+
must be the last query subtype.
|
|
20
|
+
* All other values of query subtype match exactly one event subtype
|
|
21
|
+
of the same value.
|
|
22
|
+
* Query type without subtypes is matched only by event type with no
|
|
23
|
+
subtypes.
|
|
24
|
+
|
|
25
|
+
As a consequence of aforementioned matching rules, event subtypes '*' and
|
|
26
|
+
'?' cannot be directly matched and it is advisable not to use them in event
|
|
27
|
+
types.
|
|
28
|
+
|
|
29
|
+
"""
|
|
30
|
+
is_variable = bool(query_type and query_type[-1] == '*')
|
|
31
|
+
if is_variable:
|
|
32
|
+
query_type = query_type[:-1]
|
|
33
|
+
|
|
34
|
+
if len(event_type) < len(query_type):
|
|
35
|
+
return False
|
|
36
|
+
|
|
37
|
+
if len(event_type) > len(query_type) and not is_variable:
|
|
38
|
+
return False
|
|
39
|
+
|
|
40
|
+
for i, j in zip(event_type, query_type):
|
|
41
|
+
if j != '?' and i != j:
|
|
42
|
+
return False
|
|
43
|
+
|
|
44
|
+
return True
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
from collections.abc import Collection, Iterable
|
|
2
|
+
import abc
|
|
3
|
+
import enum
|
|
4
|
+
import importlib
|
|
5
|
+
import typing
|
|
6
|
+
|
|
7
|
+
from hat import aio
|
|
8
|
+
from hat import json
|
|
9
|
+
|
|
10
|
+
from hat.event.common.common import (Event,
|
|
11
|
+
QueryParams,
|
|
12
|
+
QueryResult,
|
|
13
|
+
RegisterEvent)
|
|
14
|
+
from hat.event.common.subscription import Subscription
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class SourceType(enum.Enum):
|
|
18
|
+
EVENTER = 1
|
|
19
|
+
MODULE = 2
|
|
20
|
+
ENGINE = 3
|
|
21
|
+
SERVER = 4
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Source(typing.NamedTuple):
|
|
25
|
+
type: SourceType
|
|
26
|
+
id: int
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Engine(aio.Resource):
|
|
30
|
+
"""Engine ABC"""
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
@abc.abstractmethod
|
|
34
|
+
def server_id(self) -> int:
|
|
35
|
+
"""Event server identifier"""
|
|
36
|
+
|
|
37
|
+
@abc.abstractmethod
|
|
38
|
+
async def register(self,
|
|
39
|
+
source: Source,
|
|
40
|
+
events: Collection[RegisterEvent]
|
|
41
|
+
) -> Collection[Event] | None:
|
|
42
|
+
"""Register events"""
|
|
43
|
+
|
|
44
|
+
@abc.abstractmethod
|
|
45
|
+
async def query(self,
|
|
46
|
+
params: QueryParams
|
|
47
|
+
) -> QueryResult:
|
|
48
|
+
"""Query events"""
|
|
49
|
+
|
|
50
|
+
@abc.abstractmethod
|
|
51
|
+
def get_client_names(self) -> Iterable[tuple[Source, str]]:
|
|
52
|
+
"""Get client names connected to local eventer server"""
|
|
53
|
+
|
|
54
|
+
@abc.abstractmethod
|
|
55
|
+
def restart(self):
|
|
56
|
+
"""Schedule engine restart"""
|
|
57
|
+
|
|
58
|
+
@abc.abstractmethod
|
|
59
|
+
def reset_monitor_ready(self):
|
|
60
|
+
"""Schedule reseting of monitor component's ready flag"""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class Module(aio.Resource):
|
|
64
|
+
"""Module ABC"""
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
@abc.abstractmethod
|
|
68
|
+
def subscription(self) -> Subscription:
|
|
69
|
+
"""Subscribed event types filter.
|
|
70
|
+
|
|
71
|
+
`subscription` is constant during module's lifetime.
|
|
72
|
+
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
async def on_session_start(self, session_id: int):
|
|
76
|
+
"""Called on start of a session, identified by session_id.
|
|
77
|
+
|
|
78
|
+
This method can be coroutine or regular function.
|
|
79
|
+
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
async def on_session_stop(self, session_id: int):
|
|
83
|
+
"""Called on stop of a session, identified by session_id.
|
|
84
|
+
|
|
85
|
+
This method can be coroutine or regular function.
|
|
86
|
+
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
@abc.abstractmethod
|
|
90
|
+
async def process(self,
|
|
91
|
+
source: Source,
|
|
92
|
+
event: Event
|
|
93
|
+
) -> Iterable[RegisterEvent] | None:
|
|
94
|
+
"""Process new session event.
|
|
95
|
+
|
|
96
|
+
Provided event is matched by modules subscription filter.
|
|
97
|
+
|
|
98
|
+
Processing of session event can result in registration of
|
|
99
|
+
new register events.
|
|
100
|
+
|
|
101
|
+
Single module session process is always called sequentially.
|
|
102
|
+
|
|
103
|
+
This method can be coroutine or regular function.
|
|
104
|
+
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
ModuleConf: typing.TypeAlias = json.Data
|
|
109
|
+
"""Module configuration"""
|
|
110
|
+
|
|
111
|
+
CreateModule: typing.TypeAlias = aio.AsyncCallable[
|
|
112
|
+
[ModuleConf, Engine, Source],
|
|
113
|
+
Module]
|
|
114
|
+
"""Create module callable"""
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class ModuleInfo(typing.NamedTuple):
|
|
118
|
+
"""Module info
|
|
119
|
+
|
|
120
|
+
Module is implemented as python module which is dynamically imported.
|
|
121
|
+
It is expected that this module contains `info` which is instance of
|
|
122
|
+
`ModuleInfo`.
|
|
123
|
+
|
|
124
|
+
If module defines JSON schema repository and JSON schema id, JSON schema
|
|
125
|
+
repository will be used for additional validation of module configuration
|
|
126
|
+
with JSON schema id.
|
|
127
|
+
|
|
128
|
+
"""
|
|
129
|
+
create: CreateModule
|
|
130
|
+
json_schema_id: str | None = None
|
|
131
|
+
json_schema_repo: json.SchemaRepository | None = None
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def import_module_info(py_module_str: str) -> ModuleInfo:
|
|
135
|
+
"""Import module info"""
|
|
136
|
+
py_module = importlib.import_module(py_module_str)
|
|
137
|
+
info = py_module.info
|
|
138
|
+
|
|
139
|
+
if not isinstance(info, ModuleInfo):
|
|
140
|
+
raise Exception('invalid module implementation')
|
|
141
|
+
|
|
142
|
+
return info
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
[{"name": "HatEventer", "type_defs": {"MsgInitReq": {"name": "MsgInitReq", "args": [], "type": {"module": null, "name": "Record", "entries": [{"name": "clientName", "type": {"module": null, "name": "String", "entries": [], "args": []}}, {"name": "clientToken", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "String", "entries": [], "args": []}]}}, {"name": "subscriptions", "type": {"module": null, "name": "Array", "entries": [], "args": [{"module": null, "name": "EventType", "entries": [], "args": []}]}}, {"name": "serverId", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "Integer", "entries": [], "args": []}]}}, {"name": "persisted", "type": {"module": null, "name": "Boolean", "entries": [], "args": []}}], "args": []}}, "MsgInitRes": {"name": "MsgInitRes", "args": [], "type": {"module": null, "name": "Choice", "entries": [{"name": "success", "type": {"module": null, "name": "Status", "entries": [], "args": []}}, {"name": "error", "type": {"module": null, "name": "String", "entries": [], "args": []}}], "args": []}}, "MsgStatusNotify": {"name": "MsgStatusNotify", "args": [], "type": {"module": null, "name": "Status", "entries": [], "args": []}}, "MsgEventsNotify": {"name": "MsgEventsNotify", "args": [], "type": {"module": null, "name": "Array", "entries": [], "args": [{"module": null, "name": "Event", "entries": [], "args": []}]}}, "MsgEventsAck": {"name": "MsgEventsAck", "args": [], "type": {"module": null, "name": "None", "entries": [], "args": []}}, "MsgRegisterReq": {"name": "MsgRegisterReq", "args": [], "type": {"module": null, "name": "Array", "entries": [], "args": [{"module": null, "name": "RegisterEvent", "entries": [], "args": []}]}}, "MsgRegisterRes": {"name": "MsgRegisterRes", "args": [], "type": {"module": null, "name": "Choice", "entries": [{"name": "events", "type": {"module": null, "name": "Array", "entries": [], "args": [{"module": null, "name": "Event", "entries": [], "args": []}]}}, {"name": "failure", "type": {"module": null, "name": "None", "entries": [], "args": []}}], "args": []}}, "MsgQueryReq": {"name": "MsgQueryReq", "args": [], "type": {"module": null, "name": "QueryParams", "entries": [], "args": []}}, "MsgQueryRes": {"name": "MsgQueryRes", "args": [], "type": {"module": null, "name": "QueryResult", "entries": [], "args": []}}, "Status": {"name": "Status", "args": [], "type": {"module": null, "name": "Choice", "entries": [{"name": "standby", "type": {"module": null, "name": "None", "entries": [], "args": []}}, {"name": "starting", "type": {"module": null, "name": "None", "entries": [], "args": []}}, {"name": "operational", "type": {"module": null, "name": "None", "entries": [], "args": []}}, {"name": "stopping", "type": {"module": null, "name": "None", "entries": [], "args": []}}], "args": []}}, "Timestamp": {"name": "Timestamp", "args": [], "type": {"module": null, "name": "Record", "entries": [{"name": "s", "type": {"module": null, "name": "Integer", "entries": [], "args": []}}, {"name": "us", "type": {"module": null, "name": "Integer", "entries": [], "args": []}}], "args": []}}, "EventId": {"name": "EventId", "args": [], "type": {"module": null, "name": "Record", "entries": [{"name": "server", "type": {"module": null, "name": "Integer", "entries": [], "args": []}}, {"name": "session", "type": {"module": null, "name": "Integer", "entries": [], "args": []}}, {"name": "instance", "type": {"module": null, "name": "Integer", "entries": [], "args": []}}], "args": []}}, "Order": {"name": "Order", "args": [], "type": {"module": null, "name": "Choice", "entries": [{"name": "descending", "type": {"module": null, "name": "None", "entries": [], "args": []}}, {"name": "ascending", "type": {"module": null, "name": "None", "entries": [], "args": []}}], "args": []}}, "OrderBy": {"name": "OrderBy", "args": [], "type": {"module": null, "name": "Choice", "entries": [{"name": "timestamp", "type": {"module": null, "name": "None", "entries": [], "args": []}}, {"name": "sourceTimestamp", "type": {"module": null, "name": "None", "entries": [], "args": []}}], "args": []}}, "EventType": {"name": "EventType", "args": [], "type": {"module": null, "name": "Array", "entries": [], "args": [{"module": null, "name": "String", "entries": [], "args": []}]}}, "EventPayload": {"name": "EventPayload", "args": [], "type": {"module": null, "name": "Choice", "entries": [{"name": "binary", "type": {"module": null, "name": "EventPayloadBinary", "entries": [], "args": []}}, {"name": "json", "type": {"module": null, "name": "EventPayloadJson", "entries": [], "args": []}}], "args": []}}, "EventPayloadBinary": {"name": "EventPayloadBinary", "args": [], "type": {"module": null, "name": "Record", "entries": [{"name": "type", "type": {"module": null, "name": "String", "entries": [], "args": []}}, {"name": "data", "type": {"module": null, "name": "Bytes", "entries": [], "args": []}}], "args": []}}, "EventPayloadJson": {"name": "EventPayloadJson", "args": [], "type": {"module": null, "name": "String", "entries": [], "args": []}}, "Event": {"name": "Event", "args": [], "type": {"module": null, "name": "Record", "entries": [{"name": "id", "type": {"module": null, "name": "EventId", "entries": [], "args": []}}, {"name": "type", "type": {"module": null, "name": "EventType", "entries": [], "args": []}}, {"name": "timestamp", "type": {"module": null, "name": "Timestamp", "entries": [], "args": []}}, {"name": "sourceTimestamp", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "Timestamp", "entries": [], "args": []}]}}, {"name": "payload", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "EventPayload", "entries": [], "args": []}]}}], "args": []}}, "RegisterEvent": {"name": "RegisterEvent", "args": [], "type": {"module": null, "name": "Record", "entries": [{"name": "type", "type": {"module": null, "name": "EventType", "entries": [], "args": []}}, {"name": "sourceTimestamp", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "Timestamp", "entries": [], "args": []}]}}, {"name": "payload", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "EventPayload", "entries": [], "args": []}]}}], "args": []}}, "QueryParams": {"name": "QueryParams", "args": [], "type": {"module": null, "name": "Choice", "entries": [{"name": "latest", "type": {"module": null, "name": "QueryLatestParams", "entries": [], "args": []}}, {"name": "timeseries", "type": {"module": null, "name": "QueryTimeseriesParams", "entries": [], "args": []}}, {"name": "server", "type": {"module": null, "name": "QueryServerParams", "entries": [], "args": []}}], "args": []}}, "QueryLatestParams": {"name": "QueryLatestParams", "args": [], "type": {"module": null, "name": "Record", "entries": [{"name": "eventTypes", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "Array", "entries": [], "args": [{"module": null, "name": "EventType", "entries": [], "args": []}]}]}}], "args": []}}, "QueryTimeseriesParams": {"name": "QueryTimeseriesParams", "args": [], "type": {"module": null, "name": "Record", "entries": [{"name": "eventTypes", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "Array", "entries": [], "args": [{"module": null, "name": "EventType", "entries": [], "args": []}]}]}}, {"name": "tFrom", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "Timestamp", "entries": [], "args": []}]}}, {"name": "tTo", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "Timestamp", "entries": [], "args": []}]}}, {"name": "sourceTFrom", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "Timestamp", "entries": [], "args": []}]}}, {"name": "sourceTTo", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "Timestamp", "entries": [], "args": []}]}}, {"name": "order", "type": {"module": null, "name": "Order", "entries": [], "args": []}}, {"name": "orderBy", "type": {"module": null, "name": "OrderBy", "entries": [], "args": []}}, {"name": "maxResults", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "Integer", "entries": [], "args": []}]}}, {"name": "lastEventId", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "EventId", "entries": [], "args": []}]}}], "args": []}}, "QueryServerParams": {"name": "QueryServerParams", "args": [], "type": {"module": null, "name": "Record", "entries": [{"name": "serverId", "type": {"module": null, "name": "Integer", "entries": [], "args": []}}, {"name": "persisted", "type": {"module": null, "name": "Boolean", "entries": [], "args": []}}, {"name": "maxResults", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "Integer", "entries": [], "args": []}]}}, {"name": "lastEventId", "type": {"module": null, "name": "Optional", "entries": [], "args": [{"module": null, "name": "EventId", "entries": [], "args": []}]}}], "args": []}}, "QueryResult": {"name": "QueryResult", "args": [], "type": {"module": null, "name": "Record", "entries": [{"name": "events", "type": {"module": null, "name": "Array", "entries": [], "args": [{"module": null, "name": "Event", "entries": [], "args": []}]}}, {"name": "moreFollows", "type": {"module": null, "name": "Boolean", "entries": [], "args": []}}], "args": []}}}}, {"name": "HatEventAdminer", "type_defs": {"MsgGetLogConfReq": {"name": "MsgGetLogConfReq", "args": [], "type": {"module": null, "name": "None", "entries": [], "args": []}}, "MsgGetLogConfRes": {"name": "MsgGetLogConfRes", "args": [], "type": {"module": null, "name": "Response", "entries": [], "args": [{"module": null, "name": "String", "entries": [], "args": []}]}}, "MsgSetLogConfReq": {"name": "MsgSetLogConfReq", "args": [], "type": {"module": null, "name": "String", "entries": [], "args": []}}, "MsgSetLogConfRes": {"name": "MsgSetLogConfRes", "args": [], "type": {"module": null, "name": "Response", "entries": [], "args": [{"module": null, "name": "None", "entries": [], "args": []}]}}, "Response": {"name": "Response", "args": ["T"], "type": {"module": null, "name": "Choice", "entries": [{"name": "success", "type": {"module": null, "name": "T", "entries": [], "args": []}}, {"name": "error", "type": {"module": null, "name": "String", "entries": [], "args": []}}], "args": []}}}}]
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from collections.abc import Iterable
|
|
2
|
+
|
|
3
|
+
from hat.event.common.common import EventType
|
|
4
|
+
from hat.event.common.subscription.common import Subscription
|
|
5
|
+
from hat.event.common.subscription.pysubscription import PySubscription
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
from hat.event.common.subscription.csubscription import CSubscription
|
|
9
|
+
|
|
10
|
+
except ImportError:
|
|
11
|
+
CSubscription = None
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
__all__ = ['Subscription',
|
|
15
|
+
'create_subscription']
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def create_subscription(query_types: Iterable[EventType]) -> Subscription:
|
|
19
|
+
if CSubscription is not None:
|
|
20
|
+
return CSubscription(query_types)
|
|
21
|
+
|
|
22
|
+
return PySubscription(query_types)
|
|
Binary file
|