hat-event 0.9.27__cp310.cp311.cp312.cp313-abi3-win_amd64.whl

hat/event/__init__.py

@@ -0,0 +1 @@
+"""Event Server and communication libraries"""

hat/event/adminer/__init__.py

@@ -0,0 +1,18 @@
+"""Event adminer communication protocol"""
+
+from hat.event.adminer.client import (AdminerError,
+                                      connect,
+                                      Client)
+from hat.event.adminer.server import (GetLogConfCb,
+                                      SetLogConfCb,
+                                      listen,
+                                      Server)
+
+
+__all__ = ['AdminerError',
+           'connect',
+           'Client',
+           'GetLogConfCb',
+           'SetLogConfCb',
+           'listen',
+           'Server']

hat/event/adminer/client.py

@@ -0,0 +1,124 @@
+import asyncio
+import logging
+
+from hat import aio
+from hat import json
+from hat.drivers import chatter
+from hat.drivers import tcp
+
+from hat.event.eventer import common
+
+
+mlog: logging.Logger = logging.getLogger(__name__)
+"""Module logger"""
+
+
+class AdminerError(Exception):
+    """Errors reported by Event Adminer Server"""
+
+
+async def connect(addr: tcp.Address,
+                  **kwargs
+                  ) -> 'Client':
+    """Connect to Event Adminer Server
+
+    Additional arguments are passed to `hat.chatter.connect` coroutine.
+
+    """
+    client = Client()
+    client._loop = asyncio.get_running_loop()
+    client._conv_msg_type_futures = {}
+
+    client._conn = await chatter.connect(addr, **kwargs)
+
+    try:
+        client.async_group.spawn(client._receive_loop)
+
+    except BaseException:
+        await aio.uncancellable(client.async_close())
+        raise
+
+    return client
+
+
+class Client(aio.Resource):
+    """Event adminer client
+
+    For creating new client see `connect` coroutine.
+
+    """
+
+    @property
+    def async_group(self) -> aio.Group:
+        """Async group"""
+        return self._conn.async_group
+
+    async def get_log_conf(self) -> json.Data:
+        """Get logging configuration"""
+        data = await self._send(
+            req_msg_type='HatEventAdminer.MsgGetLogConfReq',
+            req_msg_data=None,
+            res_msg_type='HatEventAdminer.MsgGetLogConfRes')
+
+        return json.decode(data)
+
+    async def set_log_conf(self, conf: json.Data):
+        """Set logging configuration"""
+        await self._send(req_msg_type='HatEventAdminer.MsgSetLogConfReq',
+                         req_msg_data=json.encode(conf),
+                         res_msg_type='HatEventAdminer.MsgSetLogConfRes')
+
+    async def _send(self, req_msg_type, req_msg_data, res_msg_type):
+        conv = await common.send_msg(
+            conn=self._conn,
+            msg_type=req_msg_type,
+            msg_data=req_msg_data,
+            last=False)
+
+        if not self.is_open:
+            raise ConnectionError()
+
+        future = self._loop.create_future()
+        self._conv_msg_type_futures[conv] = res_msg_type, future
+
+        try:
+            return await future
+
+        finally:
+            self._conv_msg_type_futures.pop(conv, None)
+
+    async def _receive_loop(self):
+        mlog.debug("starting receive loop")
+        try:
+            while True:
+                mlog.debug("waiting for incoming message")
+                msg, msg_type, msg_data = await common.receive_msg(self._conn)
+
+                mlog.debug(f"received message {msg_type}")
+
+                res_msg_type, future = self._conv_msg_type_futures.get(
+                    msg.conv, (None, None))
+                if not future or future.done():
+                    return
+
+                if res_msg_type != msg_type:
+                    raise Exception('invalid response message type')
+
+                if msg_data[0] == 'error':
+                    future.set_exception(AdminerError(msg_data[1]))
+
+                future.set_result(msg_data[1])
+
+        except ConnectionError:
+            pass
+
+        except Exception as e:
+            mlog.error("read loop error: %s", e, exc_info=e)
+
+        finally:
+            mlog.debug("stopping receive loop")
+            self.close()
+
+            for _, future in self._conv_msg_type_futures.values():
+                if not future.done():
+                    future.set_exception(ConnectionError())

hat/event/adminer/common.py

@@ -0,0 +1,27 @@
+from hat.event.common import *  # NOQA
+
+import typing
+
+from hat import sbs
+from hat.drivers import chatter
+
+from hat.event.common import sbs_repo
+
+
+MsgType: typing.TypeAlias = str
+
+
+async def send_msg(conn: chatter.Connection,
+                   msg_type: MsgType,
+                   msg_data: sbs.Data,
+                   **kwargs
+                   ) -> chatter.Conversation:
+    msg = sbs_repo.encode(msg_type, msg_data)
+    return await conn.send(chatter.Data(msg_type, msg), **kwargs)
+
+
+async def receive_msg(conn: chatter.Connection
+                      ) -> tuple[chatter.Msg, MsgType, sbs.Data]:
+    msg = await conn.receive()
+    msg_data = sbs_repo.decode(msg.data.type, msg.data.data)
+    return msg, msg.data.type, msg_data

hat/event/adminer/server.py

@@ -0,0 +1,111 @@
+import logging
+import typing
+
+from hat import aio
+from hat import json
+from hat.drivers import chatter
+from hat.drivers import tcp
+
+from hat.event.eventer import common
+
+
+mlog: logging.Logger = logging.getLogger(__name__)
+"""Module logger"""
+
+GetLogConfCb: typing.TypeAlias = aio.AsyncCallable[[None], json.Data]
+"""Get logging configuratio callback"""
+
+SetLogConfCb: typing.TypeAlias = aio.AsyncCallable[[json.Data], None]
+"""Set logging configuratio callback"""
+
+
+async def listen(addr: tcp.Address,
+                 *,
+                 get_log_conf_cb: GetLogConfCb | None = None,
+                 set_log_conf_cb: SetLogConfCb | None = None,
+                 **kwargs
+                 ) -> 'Server':
+    """Create listening Event Adminer Server instance"""
+    server = Server()
+    server._get_log_conf_cb = get_log_conf_cb
+    server._set_log_conf_cb = set_log_conf_cb
+
+    server._srv = await chatter.listen(server._connection_loop, addr, **kwargs)
+    mlog.debug("listening on %s", addr)
+
+    return server
+
+
+class Server(aio.Resource):
+
+    @property
+    def async_group(self) -> aio.Group:
+        """Async group"""
+        return self._srv.async_group
+
+    async def _connection_loop(self, conn):
+        mlog.debug("starting connection loop")
+        try:
+            while True:
+                mlog.debug("waiting for incomming messages")
+                msg, msg_type, msg_data = await common.receive_msg(conn)
+
+                mlog.debug(f"received message {msg_type}")
+
+                if msg_type == 'HatEventAdminer.MsgGetLogConfReq':
+                    await self._process_msg_get_log_conf(
+                        conn=conn,
+                        conv=msg.conv,
+                        req_msg_data=msg_data)
+
+                elif msg_type == 'HatEventAdminer.MsgSetLogConfReq':
+                    await self._process_msg_set_log_conf(
+                        conn=conn,
+                        conv=msg.conv,
+                        req_msg_data=msg_data)
+
+                else:
+                    raise Exception('unsupported message type')
+
+        except ConnectionError:
+            pass
+
+        except Exception as e:
+            mlog.error("on connection error: %s", e, exc_info=e)
+
+        finally:
+            mlog.debug("stopping connection loop")
+            conn.close()
+
+    async def _process_msg_get_log_conf(self, conn, conv, req_msg_data):
+        try:
+            if not self._get_log_conf_cb:
+                raise Exception('not implemented')
+
+            result = await aio.call(self._get_log_conf_cb)
+
+            res_msg_data = 'success', json.encode(result)
+
+        except Exception as e:
+            res_msg_data = 'error', str(e)
+
+        await common.send_msg(
+            conn, 'HatEventAdminer.MsgGetLogConfRes', res_msg_data,
+            conv=conv)
+
+    async def _process_msg_set_log_conf(self, conn, conv, req_msg_data):
+        try:
+            if not self._set_log_conf_cb:
+                raise Exception('not implemented')
+
+            conf = json.decode(req_msg_data)
+            await aio.call(self._set_log_conf_cb, conf)
+
+            res_msg_data = 'success', None
+
+        except Exception as e:
+            res_msg_data = 'error', str(e)
+
+        await common.send_msg(
+            conn, 'HatEventAdminer.MsgSetLogConfRes', res_msg_data,
+            conv=conv)

hat/event/backends/dummy.py

@@ -0,0 +1,49 @@
+"""Dummy backend
+
+Simple backend which returns constant values where:
+
+    * `DummyBackend.get_last_event_id` returns session and instance ``0``
+    * `DummyBackend.register` returns input arguments
+    * `DummyBackend.query` returns ``QueryResult([], False)``
+
+Registered and flushed events callback is called on every register.
+
+"""
+
+from hat import aio
+
+from hat.event import common
+
+
+class DummyBackend(common.Backend):
+
+    def __init__(self, conf, registered_events_cb, flushed_events_cb):
+        self._registered_events_cbs = registered_events_cb
+        self._flushed_events_cbs = flushed_events_cb
+        self._async_group = aio.Group()
+
+    @property
+    def async_group(self):
+        return self._async_group
+
+    async def get_last_event_id(self, server_id):
+        return common.EventId(server_id, 0, 0)
+
+    async def register(self, events):
+        if self._registered_events_cbs:
+            await aio.call(self._registered_events_cbs, events)
+
+        if self._flushed_events_cbs:
+            await aio.call(self._flushed_events_cbs, events)
+
+        return events
+
+    async def query(self, params):
+        return common.QueryResult(events=[],
+                                  more_follows=False)
+
+    async def flush(self):
+        pass
+
+
+info = common.BackendInfo(DummyBackend)

hat/event/backends/lmdb/__init__.py

@@ -0,0 +1,9 @@
+"""LMDB backend"""
+
+from hat.event.backends.lmdb import common
+from hat.event.backends.lmdb.backend import create
+
+
+info = common.BackendInfo(create=create,
+                          json_schema_id='hat-event://backends/lmdb.yaml',
+                          json_schema_repo=common.json_schema_repo)

hat/event/backends/lmdb/backend.py

@@ -0,0 +1,319 @@
+from collections.abc import Collection
+from pathlib import Path
+import asyncio
+import collections
+import contextlib
+import logging
+import typing
+
+from hat import aio
+from hat import json
+
+from hat.event.backends.lmdb import common
+from hat.event.backends.lmdb import environment
+from hat.event.backends.lmdb import latestdb
+from hat.event.backends.lmdb import refdb
+from hat.event.backends.lmdb import systemdb
+from hat.event.backends.lmdb import timeseriesdb
+from hat.event.backends.lmdb.conditions import Conditions
+
+
+mlog = logging.getLogger(__name__)
+
+cleanup_max_results = 1024
+
+flush_queue_size = 4096
+
+max_registered_count = 1024 * 256
+
+version = '0.9'
+
+
+class Databases(typing.NamedTuple):
+    system: systemdb.SystemDb
+    latest: latestdb.LatestDb
+    timeseries: timeseriesdb.TimeseriesDb
+    ref: refdb.RefDb
+
+
+class Changes(typing.NamedTuple):
+    system: systemdb.Changes
+    latest: latestdb.Changes
+    timeseries: timeseriesdb.Changes
+    ref: refdb.Changes
+
+
+async def create(conf: json.Data,
+                 registered_events_cb: common.BackendRegisteredEventsCb | None,
+                 flushed_events_cb: common.BackendFlushedEventsCb | None
+                 ) -> 'LmdbBackend':
+    backend = LmdbBackend()
+    backend._registered_events_cb = registered_events_cb
+    backend._flushed_events_cb = flushed_events_cb
+    backend._conditions = Conditions(conf['conditions'])
+    backend._loop = asyncio.get_running_loop()
+    backend._flush_queue = aio.Queue(flush_queue_size)
+    backend._registered_count = 0
+    backend._registered_queue = collections.deque()
+    backend._async_group = aio.Group()
+
+    backend._env = await environment.create(Path(conf['db_path']))
+    backend.async_group.spawn(aio.call_on_done, backend._env.wait_closing(),
+                              backend.close)
+
+    try:
+        latest_subscription = common.create_subscription(
+            tuple(i) for i in conf['latest']['subscriptions'])
+
+        timeseries_partitions = (
+            timeseriesdb.Partition(
+                order_by=common.OrderBy[i['order_by']],
+                subscription=common.create_subscription(
+                    tuple(event_type) for event_type in i['subscriptions']),
+                limit=(
+                    timeseriesdb.Limit(
+                        min_entries=i['limit'].get('min_entries'),
+                        max_entries=i['limit'].get('max_entries'),
+                        duration=i['limit'].get('duration'),
+                        size=i['limit'].get('size'))
+                    if 'limit' in i else None))
+            for i in conf['timeseries'])
+
+        backend._dbs = await backend._env.execute(
+            _ext_create_dbs, backend._env, conf['identifier'],
+            backend._conditions, latest_subscription, timeseries_partitions)
+
+        backend.async_group.spawn(backend._flush_loop, conf['flush_period'])
+        backend.async_group.spawn(backend._cleanup_loop,
+                                  conf['cleanup_period'])
+
+    except BaseException:
+        await aio.uncancellable(backend._env.async_close())
+        raise
+
+    return backend
+
+
+def _ext_create_dbs(env, identifier, conditions, latest_subscription,
+                    timeseries_partitions):
+    with env.ext_begin(write=True) as txn:
+        system_db = systemdb.ext_create(env, txn, version, identifier)
+        latest_db = latestdb.ext_create(env, txn, conditions,
+                                        latest_subscription)
+        timeseries_db = timeseriesdb.ext_create(env, txn, conditions,
+                                                timeseries_partitions)
+        ref_db = refdb.RefDb(env)
+
+    return Databases(system=system_db,
+                     latest=latest_db,
+                     timeseries=timeseries_db,
+                     ref=ref_db)
+
+
+class LmdbBackend(common.Backend):
+
+    @property
+    def async_group(self) -> aio.Group:
+        return self._async_group
+
+    async def get_last_event_id(self,
+                                server_id: int
+                                ) -> common.EventId:
+        if not self.is_open:
+            raise common.BackendClosedError()
+
+        return self._dbs.system.get_last_event_id(server_id)
+
+    async def register(self,
+                       events: Collection[common.Event]
+                       ) -> Collection[common.Event] | None:
+        if not self.is_open:
+            raise common.BackendClosedError()
+
+        for event in events:
+            server_id = event.id.server
+
+            last_event_id = self._dbs.system.get_last_event_id(server_id)
+            last_timestamp = self._dbs.system.get_last_timestamp(server_id)
+
+            if last_event_id >= event.id:
+                mlog.warning("event registration skipped: invalid event id")
+                continue
+
+            if last_timestamp > event.timestamp:
+                mlog.warning("event registration skipped: invalid timestamp")
+                continue
+
+            if not self._conditions.matches(event):
+                mlog.warning("event registration skipped: invalid conditions")
+                continue
+
+            refs = collections.deque()
+
+            latest_result = self._dbs.latest.add(event)
+
+            if latest_result.added_ref:
+                refs.append(latest_result.added_ref)
+
+            if latest_result.removed_ref:
+                self._dbs.ref.remove(*latest_result.removed_ref)
+
+            refs.extend(self._dbs.timeseries.add(event))
+
+            if not refs:
+                continue
+
+            self._dbs.ref.add(event, refs)
+            self._dbs.system.set_last_event_id(event.id)
+            self._dbs.system.set_last_timestamp(server_id, event.timestamp)
+
+        self._registered_queue.append(events)
+        self._registered_count += len(events)
+
+        if self._registered_count > max_registered_count:
+            await self._flush_queue.put(None)
+
+        if self._registered_events_cb:
+            await aio.call(self._registered_events_cb, events)
+
+        return events
+
+    async def query(self,
+                    params: common.QueryParams
+                    ) -> common.QueryResult:
+        if not self.is_open:
+            raise common.BackendClosedError()
+
+        if isinstance(params, common.QueryLatestParams):
+            return self._dbs.latest.query(params)
+
+        if isinstance(params, common.QueryTimeseriesParams):
+            return await self._dbs.timeseries.query(params)
+
+        if isinstance(params, common.QueryServerParams):
+            return await self._dbs.ref.query(params)
+
+        raise ValueError('unsupported params type')
+
+    async def flush(self):
+        try:
+            future = self._loop.create_future()
+            await self._flush_queue.put(future)
+            await future
+
+        except aio.QueueClosedError:
+            raise common.BackendClosedError()
+
+    async def _flush_loop(self, flush_period):
+        futures = collections.deque()
+
+        async def cleanup():
+            with contextlib.suppress(Exception):
+                await self._flush()
+
+            await self._env.async_close()
+
+        try:
+            while True:
+                try:
+                    future = await aio.wait_for(self._flush_queue.get(),
+                                                flush_period)
+                    futures.append(future)
+
+                except asyncio.TimeoutError:
+                    pass
+
+                except aio.CancelledWithResultError as e:
+                    if e.result:
+                        futures.append(e.result)
+
+                    raise
+
+                while not self._flush_queue.empty():
+                    futures.append(self._flush_queue.get_nowait())
+
+                await aio.uncancellable(self._flush())
+
+                while futures:
+                    future = futures.popleft()
+                    if future and not future.done():
+                        future.set_result(None)
+
+        except Exception as e:
+            mlog.error('backend flush error: %s', e, exc_info=e)
+
+        finally:
+            self.close()
+            self._flush_queue.close()
+
+            while not self._flush_queue.empty():
+                futures.append(self._flush_queue.get_nowait())
+
+            for future in futures:
+                if future and not future.done():
+                    future.set_exception(common.BackendClosedError())
+
+            await aio.uncancellable(cleanup())
+
+    async def _cleanup_loop(self, cleanup_period):
+        try:
+            while True:
+                await asyncio.sleep(0)
+
+                repeat = await self._env.execute(_ext_cleanup, self._env,
+                                                 self._dbs, common.now())
+                if repeat:
+                    continue
+
+                await asyncio.sleep(cleanup_period)
+
+        except Exception as e:
+            mlog.error('backend cleanup error: %s', e, exc_info=e)
+
+        finally:
+            self.close()
+
+    async def _flush(self):
+        if not self._env.is_open:
+            return
+
+        self._registered_count = 0
+        registered_queue, self._registered_queue = (self._registered_queue,
+                                                    collections.deque())
+
+        changes = Changes(system=self._dbs.system.create_changes(),
+                          latest=self._dbs.latest.create_changes(),
+                          timeseries=self._dbs.timeseries.create_changes(),
+                          ref=self._dbs.ref.create_changes())
+
+        # TODO lock period between create_changes and locking executor
+        #      (timeseries and ref must write changes before new queries are
+        #      allowed)
+
+        await self._env.execute(_ext_flush, self._env, self._dbs, changes)
+
+        if not self._flushed_events_cb:
+            return
+
+        while registered_queue:
+            events = registered_queue.popleft()
+            await aio.call(self._flushed_events_cb, events)
+
+
+def _ext_flush(env, dbs, changes):
+    with env.ext_begin(write=True) as txn:
+        dbs.system.ext_write(txn, changes.system)
+        dbs.latest.ext_write(txn, changes.latest)
+        dbs.timeseries.ext_write(txn, changes.timeseries)
+        dbs.ref.ext_write(txn, changes.ref)
+
+
+def _ext_cleanup(env, dbs, now):
+    with env.ext_begin(write=True) as txn:
+        result = dbs.timeseries.ext_cleanup(txn, now, cleanup_max_results)
+        if not result:
+            return False
+
+        dbs.ref.ext_cleanup(txn, result)
+
+    return len(result) >= cleanup_max_results