hat-event 0.9.27__cp310.cp311.cp312.cp313-abi3-win_amd64.whl

hat/event/common/__init__.py

@@ -0,0 +1,144 @@
+"""Common functionality shared between clients and event server"""
+
+import time
+
+from hat.event.common.backend import (BackendClosedError,
+                                      Backend,
+                                      BackendConf,
+                                      BackendRegisteredEventsCb,
+                                      BackendFlushedEventsCb,
+                                      CreateBackend,
+                                      BackendInfo,
+                                      import_backend_info)
+from hat.event.common.collection import (EventTypeCollection,
+                                         ListEventTypeCollection,
+                                         TreeEventTypeCollection)
+from hat.event.common.common import (json_schema_repo,
+                                     sbs_repo,
+                                     ServerId,
+                                     SessionId,
+                                     InstanceId,
+                                     EventTypeSegment,
+                                     EventType,
+                                     Timestamp,
+                                     min_timestamp,
+                                     max_timestamp,
+                                     Status,
+                                     Order,
+                                     OrderBy,
+                                     EventId,
+                                     EventPayloadBinary,
+                                     EventPayloadJson,
+                                     EventPayload,
+                                     Event,
+                                     RegisterEvent,
+                                     QueryLatestParams,
+                                     QueryTimeseriesParams,
+                                     QueryServerParams,
+                                     QueryParams,
+                                     QueryResult)
+from hat.event.common.encoder import (timestamp_to_bytes,
+                                      timestamp_from_bytes,
+                                      timestamp_to_float,
+                                      timestamp_from_float,
+                                      timestamp_to_datetime,
+                                      timestamp_from_datetime,
+                                      timestamp_to_sbs,
+                                      timestamp_from_sbs,
+                                      status_to_sbs,
+                                      status_from_sbs,
+                                      event_to_sbs,
+                                      event_from_sbs,
+                                      register_event_to_sbs,
+                                      register_event_from_sbs,
+                                      query_params_to_sbs,
+                                      query_params_from_sbs,
+                                      query_result_to_sbs,
+                                      query_result_from_sbs,
+                                      event_payload_to_sbs,
+                                      event_payload_from_sbs)
+from hat.event.common.matches import matches_query_type
+from hat.event.common.module import (SourceType,
+                                     Source,
+                                     Engine,
+                                     Module,
+                                     ModuleConf,
+                                     CreateModule,
+                                     ModuleInfo,
+                                     import_module_info)
+from hat.event.common.subscription import (Subscription,
+                                           create_subscription)
+
+
+__all__ = ['BackendClosedError',
+           'Backend',
+           'BackendConf',
+           'BackendRegisteredEventsCb',
+           'BackendFlushedEventsCb',
+           'CreateBackend',
+           'BackendInfo',
+           'import_backend_info',
+           'EventTypeCollection',
+           'ListEventTypeCollection',
+           'TreeEventTypeCollection',
+           'json_schema_repo',
+           'sbs_repo',
+           'ServerId',
+           'SessionId',
+           'InstanceId',
+           'EventTypeSegment',
+           'EventType',
+           'Timestamp',
+           'min_timestamp',
+           'max_timestamp',
+           'Status',
+           'Order',
+           'OrderBy',
+           'EventId',
+           'EventPayloadBinary',
+           'EventPayloadJson',
+           'EventPayload',
+           'Event',
+           'RegisterEvent',
+           'QueryLatestParams',
+           'QueryTimeseriesParams',
+           'QueryServerParams',
+           'QueryParams',
+           'QueryResult',
+           'timestamp_to_bytes',
+           'timestamp_from_bytes',
+           'timestamp_to_float',
+           'timestamp_from_float',
+           'timestamp_to_datetime',
+           'timestamp_from_datetime',
+           'timestamp_to_sbs',
+           'timestamp_from_sbs',
+           'status_to_sbs',
+           'status_from_sbs',
+           'event_to_sbs',
+           'event_from_sbs',
+           'register_event_to_sbs',
+           'register_event_from_sbs',
+           'query_params_to_sbs',
+           'query_params_from_sbs',
+           'query_result_to_sbs',
+           'query_result_from_sbs',
+           'event_payload_to_sbs',
+           'event_payload_from_sbs',
+           'matches_query_type',
+           'SourceType',
+           'Source',
+           'Engine',
+           'Module',
+           'ModuleConf',
+           'CreateModule',
+           'ModuleInfo',
+           'import_module_info',
+           'Subscription',
+           'create_subscription',
+           'now']
+
+
+def now() -> Timestamp:
+    """Create new timestamp representing current time"""
+    return timestamp_from_float(time.time())

hat/event/common/backend.py

@@ -0,0 +1,91 @@
+from collections.abc import Collection
+import abc
+import importlib
+import typing
+
+from hat import aio
+from hat import json
+
+from hat.event.common.common import (EventId,
+                                     Event,
+                                     QueryParams,
+                                     QueryResult)
+
+
+class BackendClosedError(Exception):
+    """Backend closed"""
+
+
+class Backend(aio.Resource):
+    """Backend ABC"""
+
+    @abc.abstractmethod
+    async def get_last_event_id(self,
+                                server_id: int
+                                ) -> EventId:
+        """Get last registered event id associated with server id"""
+
+    @abc.abstractmethod
+    async def register(self,
+                       events: Collection[Event]
+                       ) -> Collection[Event] | None:
+        """Register events"""
+
+    @abc.abstractmethod
+    async def query(self,
+                    params: QueryParams
+                    ) -> QueryResult:
+        """Query events"""
+
+    @abc.abstractmethod
+    async def flush(self):
+        """Flush internal buffers and permanently persist events"""
+
+
+BackendConf: typing.TypeAlias = json.Data
+"""Backend configuration"""
+
+BackendRegisteredEventsCb: typing.TypeAlias = aio.AsyncCallable[
+    [Collection[Event]],
+    None]
+"""Backend registered events callback"""
+
+BackendFlushedEventsCb: typing.TypeAlias = aio.AsyncCallable[
+    [Collection[Event]],
+    None]
+"""Backend flushed events callback"""
+
+CreateBackend: typing.TypeAlias = aio.AsyncCallable[
+    [BackendConf,
+     BackendRegisteredEventsCb | None,
+     BackendFlushedEventsCb | None],
+    Backend]
+"""Create backend callable"""
+
+
+class BackendInfo(typing.NamedTuple):
+    """Backend info
+
+    Backend is implemented as python module which is dynamically imported.
+    It is expected that this module contains `info` which is instance of
+    `BackendInfo`.
+
+    If backend defines JSON schema repository and JSON schema id, JSON schema
+    repository will be used for additional validation of backend configuration
+    with JSON schema id.
+
+    """
+    create: CreateBackend
+    json_schema_id: str | None = None
+    json_schema_repo: json.SchemaRepository | None = None
+
+
+def import_backend_info(py_module_str: str) -> BackendInfo:
+    """Import backend info"""
+    py_module = importlib.import_module(py_module_str)
+    info = py_module.info
+
+    if not isinstance(info, BackendInfo):
+        raise Exception('invalid backend implementation')
+
+    return info

hat/event/common/collection/__init__.py

@@ -0,0 +1,8 @@
+from hat.event.common.collection.common import EventTypeCollection
+from hat.event.common.collection.list import ListEventTypeCollection
+from hat.event.common.collection.tree import TreeEventTypeCollection
+
+
+__all__ = ['EventTypeCollection',
+           'ListEventTypeCollection',
+           'TreeEventTypeCollection']

hat/event/common/collection/common.py

@@ -0,0 +1,28 @@
+from collections.abc import Hashable, Iterable
+import abc
+import typing
+
+from hat.event.common.common import EventType
+from hat.event.common.subscription import Subscription
+
+
+T = typing.TypeVar('T', bound=Hashable)
+
+
+class EventTypeCollection(abc.ABC, typing.Generic[T]):
+
+    @abc.abstractmethod
+    def __init__(self):
+        pass
+
+    @abc.abstractmethod
+    def add(self, subscription: Subscription, value: T):
+        pass
+
+    @abc.abstractmethod
+    def remove(self, value: T):
+        pass
+
+    @abc.abstractmethod
+    def get(self, event_type: EventType) -> Iterable[T]:
+        pass

hat/event/common/collection/list.py

@@ -0,0 +1,19 @@
+from hat.event.common.collection import common
+
+
+class ListEventTypeCollection(common.EventTypeCollection):
+
+    def __init__(self):
+        self._values = {}
+
+    def add(self, subscription, value):
+        self._values[value] = (self._values[value].union(subscription)
+                               if value in self._values else subscription)
+
+    def remove(self, value):
+        self._values.pop(value, None)
+
+    def get(self, event_type):
+        for value, subscription in self._values.items():
+            if subscription.matches(event_type):
+                yield value

hat/event/common/collection/tree.py

@@ -0,0 +1,62 @@
+import collections
+import typing
+
+from hat.event.common.collection import common
+
+
+class TreeEventTypeCollection(common.EventTypeCollection):
+
+    def __init__(self):
+        self._root = _create_node()
+        self._value_nodes = collections.defaultdict(collections.deque)
+
+    def add(self, subscription, value):
+        for query_type in subscription.get_query_types():
+            node = self._root
+            rest = query_type
+
+            while rest:
+                head, rest = rest[0], rest[1:]
+                if head == '*' and rest:
+                    raise ValueError('invalid query type')
+
+                node = node.children[head]
+
+            if value in node.values:
+                return
+
+            node.values.add(value)
+            self._value_nodes[value].append(node)
+
+    def remove(self, value):
+        for node in self._value_nodes.pop(value, []):
+            node.values.remove(value)
+
+    def get(self, event_type):
+        return set(_get(self._root, event_type))
+
+
+class _Node(typing.NamedTuple):
+    values: set
+    children: collections.defaultdict[str, '_Node']
+
+
+def _create_node():
+    return _Node(set(), collections.defaultdict(_create_node))
+
+
+def _get(node, event_type):
+    if '*' in node.children:
+        yield from node.children['*'].values
+
+    if not event_type:
+        yield from node.values
+        return
+
+    head, rest = event_type[0], event_type[1:]
+
+    if head in node.children:
+        yield from _get(node.children[head], rest)
+
+    if '?' in node.children:
+        yield from _get(node.children['?'], rest)

hat/event/common/common.py

@@ -0,0 +1,176 @@
+from collections.abc import Collection
+import enum
+import importlib.resources
+import typing
+
+from hat import json
+from hat import sbs
+from hat import util
+
+
+with importlib.resources.as_file(importlib.resources.files(__package__) /
+                                 'json_schema_repo.json') as _path:
+    json_schema_repo: json.SchemaRepository = json.SchemaRepository(
+        json.json_schema_repo,
+        json.SchemaRepository.from_json(_path))
+    """JSON schema repository"""
+
+with importlib.resources.as_file(importlib.resources.files(__package__) /
+                                 'sbs_repo.json') as _path:
+    sbs_repo: sbs.Repository = sbs.Repository.from_json(_path)
+    """SBS schema repository"""
+
+ServerId: typing.TypeAlias = int
+"""Server identifier"""
+
+SessionId: typing.TypeAlias = int
+"""Session identifier"""
+
+InstanceId: typing.TypeAlias = int
+"""Event instance identifier"""
+
+EventTypeSegment: typing.TypeAlias = str
+"""Event type segment"""
+
+EventType: typing.TypeAlias = tuple[EventTypeSegment, ...]
+"""Event type"""
+
+
+class Timestamp(typing.NamedTuple):
+    s: int
+    """seconds since 1970-01-01 (can be negative)"""
+    us: int
+    """microseconds added to timestamp seconds in range [0, 1e6)"""
+
+    def add(self, s: float) -> 'Timestamp':
+        """Create new timestamp by adding seconds to existing timestamp"""
+        us = self.us + round((s - int(s)) * 1e6)
+        s = self.s + int(s)
+        return Timestamp(s=s + us // int(1e6),
+                         us=us % int(1e6))
+
+
+min_timestamp: Timestamp = Timestamp(s=-(1 << 63), us=0)
+"""Minimal byte serializable timestamp value"""
+
+max_timestamp: Timestamp = Timestamp(s=(1 << 63) - 1, us=999_999)
+"""Maximal byte serializable timestamp value"""
+
+
+class Status(enum.Enum):
+    STANDBY = 'standby'
+    STARTING = 'starting'
+    OPERATIONAL = 'operational'
+    STOPPING = 'stopping'
+
+
+class Order(enum.Enum):
+    DESCENDING = 'descending'
+    ASCENDING = 'ascending'
+
+
+class OrderBy(enum.Enum):
+    TIMESTAMP = 'timestamp'
+    SOURCE_TIMESTAMP = 'sourceTimestamp'
+
+
+class EventId(typing.NamedTuple):
+    server: ServerId
+    session: SessionId
+    instance: InstanceId
+
+
+class EventPayloadBinary(typing.NamedTuple):
+    type: str
+    data: util.Bytes
+
+
+class EventPayloadJson(typing.NamedTuple):
+    data: json.Data
+
+
+EventPayload: typing.TypeAlias = EventPayloadBinary | EventPayloadJson
+
+
+class Event(typing.NamedTuple):
+    """Event
+
+    Operators `>` and `<` test for natural order where it is assumed that
+    first operand is registered before second operand.
+
+    """
+
+    id: EventId
+    type: EventType
+    timestamp: Timestamp
+    source_timestamp: Timestamp | None
+    payload: EventPayload | None
+
+    def __lt__(self, other):
+        if not isinstance(other, Event):
+            return NotImplemented
+
+        if self.id == other.id:
+            return False
+
+        if self.id.server == other.id.server:
+            return self.id < other.id
+
+        if self.timestamp != other.timestamp:
+            return self.timestamp < other.timestamp
+
+        return True
+
+    def __gt__(self, other):
+        if not isinstance(other, Event):
+            return NotImplemented
+
+        if self.id == other.id:
+            return False
+
+        if self.id.server == other.id.server:
+            return self.id > other.id
+
+        if self.timestamp != other.timestamp:
+            return self.timestamp > other.timestamp
+
+        return False
+
+
+class RegisterEvent(typing.NamedTuple):
+    type: EventType
+    source_timestamp: Timestamp | None
+    payload: EventPayload | None
+
+
+class QueryLatestParams(typing.NamedTuple):
+    event_types: Collection[EventType] | None = None
+
+
+class QueryTimeseriesParams(typing.NamedTuple):
+    event_types: Collection[EventType] | None = None
+    t_from: Timestamp | None = None
+    t_to: Timestamp | None = None
+    source_t_from: Timestamp | None = None
+    source_t_to: Timestamp | None = None
+    order: Order = Order.DESCENDING
+    order_by: OrderBy = OrderBy.TIMESTAMP
+    max_results: int | None = None
+    last_event_id: EventId | None = None
+
+
+class QueryServerParams(typing.NamedTuple):
+    server_id: ServerId
+    persisted: bool = False
+    max_results: int | None = None
+    last_event_id: EventId | None = None
+
+
+QueryParams: typing.TypeAlias = (QueryLatestParams |
+                                 QueryTimeseriesParams |
+                                 QueryServerParams)
+
+
+class QueryResult(typing.NamedTuple):
+    events: Collection[Event]
+    more_follows: bool