hat-event 0.9.27__cp310.cp311.cp312.cp313-abi3-win_amd64.whl

hat/event/backends/lmdb/common.py

@@ -0,0 +1,277 @@
+from hat.event.common import *  # NOQA
+
+from pathlib import Path
+import enum
+import itertools
+import platform
+import struct
+import sys
+import typing
+
+import lmdb
+
+from hat import json
+from hat import util
+
+from hat.event.common import (ServerId, Event, EventId, EventType,
+                              Timestamp, EventPayloadBinary,
+                              sbs_repo, event_to_sbs, event_from_sbs)
+
+
+default_max_size: int = (1024 * 1024 * 1024 * 1024
+                         if platform.architecture()[0] == '64bit'
+                         else 2 * 1024 * 1024 * 1024)
+
+
+DbKey = typing.TypeVar('DbKey')
+DbValue = typing.TypeVar('DbValue')
+
+
+class DbType(enum.Enum):
+    SYSTEM_SETTINGS = 0
+    SYSTEM_LAST_EVENT_ID = 1
+    SYSTEM_LAST_TIMESTAMP = 2
+    REF = 3
+    LATEST_DATA = 4
+    LATEST_TYPE = 5
+    TIMESERIES_DATA = 6
+    TIMESERIES_PARTITION = 7
+    TIMESERIES_COUNT = 8
+
+
+if sys.version_info[:2] >= (3, 11):
+
+    class DbDef(typing.NamedTuple, typing.Generic[DbKey, DbValue]):
+        encode_key: typing.Callable[[DbKey], util.Bytes]
+        decode_key: typing.Callable[[util.Bytes], DbKey]
+        encode_value: typing.Callable[[DbValue], util.Bytes]
+        decode_value: typing.Callable[[util.Bytes], DbValue]
+
+    def create_db_def(key_type: typing.Type,
+                      value_type: typing.Type
+                      ) -> typing.Type[DbDef]:
+        return DbDef[key_type, value_type]
+
+else:
+
+    class DbDef(typing.NamedTuple):
+        encode_key: typing.Callable[[DbKey], util.Bytes]
+        decode_key: typing.Callable[[util.Bytes], DbKey]
+        encode_value: typing.Callable[[DbValue], util.Bytes]
+        decode_value: typing.Callable[[util.Bytes], DbValue]
+
+    def create_db_def(key_type: typing.Type,
+                      value_type: typing.Type
+                      ) -> typing.Type[DbDef]:
+        return DbDef
+
+
+EventTypeRef: typing.TypeAlias = int
+PartitionId: typing.TypeAlias = int
+TimeseriesKey: typing.TypeAlias = tuple[PartitionId, Timestamp, EventId]
+
+
+class SettingsId(enum.Enum):
+    VERSION = 0
+    IDENTIFIER = 1
+
+
+class LatestEventRef(typing.NamedTuple):
+    key: EventTypeRef
+
+
+class TimeseriesEventRef(typing.NamedTuple):
+    key: TimeseriesKey
+
+
+EventRef: typing.TypeAlias = LatestEventRef | TimeseriesEventRef
+
+
+db_defs = {
+    DbType.SYSTEM_SETTINGS: create_db_def(SettingsId, json.Data)(
+        encode_key=lambda key: _encode_uint(key.value),
+        decode_key=lambda key_bytes: SettingsId(_decode_uint(key_bytes)),
+        encode_value=lambda value: _encode_json(value),
+        decode_value=lambda value_bytes: _decode_json(value_bytes)),
+
+    DbType.SYSTEM_LAST_EVENT_ID: create_db_def(ServerId, EventId)(
+        encode_key=lambda key: _encode_uint(key),
+        decode_key=lambda key_bytes: _decode_uint(key_bytes),
+        encode_value=lambda value: _encode_event_id(value),
+        decode_value=lambda value_bytes: _decode_event_id(value_bytes)),
+
+    DbType.SYSTEM_LAST_TIMESTAMP: create_db_def(ServerId, Timestamp)(
+        encode_key=lambda key: _encode_uint(key),
+        decode_key=lambda key_bytes: _decode_uint(key_bytes),
+        encode_value=lambda value: _encode_timestamp(value),
+        decode_value=lambda value_bytes: _decode_timestamp(value_bytes)),
+
+    DbType.REF: create_db_def(EventId, set[EventRef])(
+        encode_key=lambda key: _encode_event_id(key),
+        decode_key=lambda key_bytes: _decode_event_id(key_bytes),
+        encode_value=lambda value: bytes(_encode_event_refs(value)),
+        decode_value=lambda value_bytes: set(_decode_event_refs(value_bytes))),
+
+    DbType.LATEST_DATA: create_db_def(EventTypeRef, Event)(
+        encode_key=lambda key: _encode_uint(key),
+        decode_key=lambda key_bytes: _decode_uint(key_bytes),
+        encode_value=lambda value: _encode_event(value),
+        decode_value=lambda value_bytes: _decode_event(value_bytes)),
+
+    DbType.LATEST_TYPE: create_db_def(EventTypeRef, EventType)(
+        encode_key=lambda key: _encode_uint(key),
+        decode_key=lambda key_bytes: _decode_uint(key_bytes),
+        encode_value=lambda value: _encode_json(list(value)),
+        decode_value=lambda value_bytes: tuple(_decode_json(value_bytes))),
+
+    DbType.TIMESERIES_DATA: create_db_def(TimeseriesKey, Event)(
+        encode_key=lambda key: _encode_timeseries_key(key),
+        decode_key=lambda key_bytes: _decode_timeseries_key(key_bytes),
+        encode_value=lambda value: _encode_event(value),
+        decode_value=lambda value_bytes: _decode_event(value_bytes)),
+
+    DbType.TIMESERIES_PARTITION: create_db_def(PartitionId, json.Data)(
+        encode_key=lambda key: _encode_uint(key),
+        decode_key=lambda key_bytes: _decode_uint(key_bytes),
+        encode_value=lambda value: _encode_json(value),
+        decode_value=lambda value_bytes: _decode_json(value_bytes)),
+
+    DbType.TIMESERIES_COUNT: create_db_def(PartitionId, int)(
+        encode_key=lambda key: _encode_uint(key),
+        decode_key=lambda key_bytes: _decode_uint(key_bytes),
+        encode_value=lambda value: _encode_uint(value),
+        decode_value=lambda value_bytes: _decode_uint(value_bytes))}
+
+
+def ext_create_env(path: Path,
+                   max_size: int = default_max_size,
+                   readonly: bool = False
+                   ) -> lmdb.Environment:
+    return lmdb.Environment(str(path),
+                            map_size=max_size,
+                            subdir=False,
+                            max_dbs=len(DbType),
+                            readonly=readonly)
+
+
+def ext_open_db(env: lmdb.Environment,
+                db_type: DbType,
+                create: bool = True
+                ) -> lmdb._Database:
+    return env.open_db(db_type.name.encode('utf-8'), create=create)
+
+
+def _encode_uint(value):
+    return struct.pack(">Q", value)
+
+
+def _decode_uint(value_bytes):
+    return struct.unpack(">Q", value_bytes)[0]
+
+
+def _encode_event_id(event_id):
+    return struct.pack(">QQQ", event_id.server, event_id.session,
+                       event_id.instance)
+
+
+def _decode_event_id(event_id_bytes):
+    server_id, session_id, instance_id = struct.unpack(">QQQ", event_id_bytes)
+    return EventId(server=server_id,
+                   session=session_id,
+                   instance=instance_id)
+
+
+def _encode_timestamp(timestamp):
+    return struct.pack(">QI", timestamp.s + (1 << 63), timestamp.us)
+
+
+def _decode_timestamp(timestamp_bytes):
+    s, us = struct.unpack(">QI", timestamp_bytes)
+    return Timestamp(s - (1 << 63), us)
+
+
+def _encode_event_ref(ref):
+    if isinstance(ref, LatestEventRef):
+        db_type = DbType.LATEST_DATA
+
+    elif isinstance(ref, TimeseriesEventRef):
+        db_type = DbType.TIMESERIES_DATA
+
+    else:
+        raise ValueError('unsupported event reference')
+
+    return bytes([db_type.value]) + db_defs[db_type].encode_key(ref.key)
+
+
+def _decode_event_ref(ref_bytes):
+    db_type = DbType(ref_bytes[0])
+    key = db_defs[db_type].decode_key(ref_bytes[1:])
+
+    if db_type == DbType.LATEST_DATA:
+        return LatestEventRef(key)
+
+    if db_type == DbType.TIMESERIES_DATA:
+        return TimeseriesEventRef(key)
+
+    raise ValueError('unsupported database type')
+
+
+def _encode_event_refs(refs):
+    return bytes(itertools.chain.from_iterable(_encode_event_ref(ref)
+                                               for ref in refs))
+
+
+def _decode_event_refs(refs_bytes):
+    while refs_bytes:
+        db_type = DbType(refs_bytes[0])
+
+        if db_type == DbType.LATEST_DATA:
+            ref_key_len = 8
+
+        elif db_type == DbType.TIMESERIES_DATA:
+            ref_key_len = 44
+
+        else:
+            raise ValueError('unsupported event reference')
+
+        ref, refs_bytes = (_decode_event_ref(refs_bytes[:ref_key_len+1]),
+                           refs_bytes[ref_key_len+1:])
+        yield ref
+
+
+def _encode_timeseries_key(key):
+    partition_id, timestamp, event_id = key
+    return bytes(itertools.chain(_encode_uint(partition_id),
+                                 _encode_timestamp(timestamp),
+                                 _encode_event_id(event_id)))
+
+
+def _decode_timeseries_key(key_bytes):
+    partition_id = _decode_uint(key_bytes[:8])
+    timestamp = _decode_timestamp(key_bytes[8:20])
+    event_id = _decode_event_id(key_bytes[20:])
+    return partition_id, timestamp, event_id
+
+
+def _encode_event(event):
+    event_sbs = event_to_sbs(event)
+    return sbs_repo.encode('HatEventer.Event', event_sbs)
+
+
+def _decode_event(event_bytes):
+    event_sbs = sbs_repo.decode('HatEventer.Event', event_bytes)
+    event = event_from_sbs(event_sbs)
+
+    if isinstance(event.payload, EventPayloadBinary):
+        event = event._replace(
+            payload=event.payload._replace(data=bytes(event.payload.data)))
+
+    return event
+
+
+def _encode_json(data):
+    return json.encode(data).encode('utf-8')
+
+
+def _decode_json(data_bytes):
+    return json.decode(str(data_bytes, encoding='utf-8'))

hat/event/backends/lmdb/conditions.py

@@ -0,0 +1,102 @@
+from hat import json
+
+from hat.event.backends.lmdb import common
+
+
+class Conditions:
+
+    def __init__(self, conf: json.Data):
+        self._conditions = [
+            (common.create_subscription(tuple(event_type)
+                                        for event_type in i['subscriptions']),
+             _create_condition(i['condition']))
+            for i in conf]
+
+    def matches(self, event: common.Event) -> bool:
+        for subscription, condition in self._conditions:
+            if not subscription.matches(event.type):
+                continue
+
+            if not condition.matches(event):
+                return False
+
+        return True
+
+
+def _create_condition(conf):
+    if conf['type'] == 'all':
+        return _AllCondition(conf)
+
+    if conf['type'] == 'any':
+        return _AnyCondition(conf)
+
+    if conf['type'] == 'json':
+        return _JsonCondition(conf)
+
+    raise ValueError('unsupported condition type')
+
+
+class _AllCondition:
+
+    def __init__(self, conf):
+        self._conditions = [_create_condition(i) for i in conf['conditions']]
+
+    def matches(self, event):
+        return all(condition.matches(event) for condition in self._conditions)
+
+
+class _AnyCondition:
+
+    def __init__(self, conf):
+        self._conditions = [_create_condition(i) for i in conf['conditions']]
+
+    def matches(self, event):
+        return any(condition.matches(event) for condition in self._conditions)
+
+
+class _JsonCondition:
+
+    def __init__(self, conf):
+        self._conf = conf
+
+    def matches(self, event):
+        if not isinstance(event.payload, common.EventPayloadJson):
+            return False
+
+        data_path = self._conf.get('data_path', [])
+        data = json.get(event.payload.data, data_path)
+
+        if 'data_type' in self._conf:
+            data_type = self._conf['data_type']
+
+            if data_type == 'null':
+                if data is not None:
+                    return False
+
+            elif data_type == 'boolean':
+                if not isinstance(data, bool):
+                    return False
+
+            elif data_type == 'string':
+                if not isinstance(data, str):
+                    return False
+
+            elif data_type == 'number':
+                if not (isinstance(data, float) or
+                        (isinstance(data, int) and
+                         not isinstance(data, bool))):
+                    return False
+
+            elif data_type == 'array':
+                if not isinstance(data, list):
+                    return False
+
+            elif data_type == 'object':
+                if not isinstance(data, dict):
+                    return False
+
+        if 'data_value' in self._conf:
+            if self._conf['data_value'] != data:
+                return False
+
+        return True

hat/event/backends/lmdb/convert/__main__.py

@@ -0,0 +1,8 @@
+import sys
+
+from hat.event.backends.lmdb.convert.main import main
+
+
+if __name__ == '__main__':
+    sys.argv[0] = 'hat-event-lmdb-convert'
+    sys.exit(main())

hat/event/backends/lmdb/convert/convert_v06_to_v07.py

@@ -0,0 +1,213 @@
+from pathlib import Path
+import itertools
+
+from hat.event.backends.lmdb.convert import v06
+from hat.event.backends.lmdb.convert import v07
+
+
+def convert(src_path: Path,
+            dst_path: Path):
+    with v06.create_env(src_path) as src_env:
+        src_system_db = src_env.open_db(b'system')
+        server_id = _get_server_id(src_env, src_system_db)
+
+        if server_id < 1:
+            server_id = 1
+
+        with v07.create_env(dst_path) as dst_env:
+            dst_system_db = v07.open_db(dst_env, v07.DbType.SYSTEM)
+            dst_ref_db = v07.open_db(dst_env, v07.DbType.REF)
+
+            _convert_latest(src_env=src_env,
+                            dst_env=dst_env,
+                            dst_system_db=dst_system_db,
+                            dst_ref_db=dst_ref_db,
+                            server_id=server_id)
+
+            _convert_ordered(src_env=src_env,
+                             dst_env=dst_env,
+                             dst_system_db=dst_system_db,
+                             dst_ref_db=dst_ref_db,
+                             server_id=server_id)
+
+
+def _convert_latest(src_env, dst_env, dst_system_db, dst_ref_db, server_id):
+    src_latest_db = src_env.open_db(b'latest')
+
+    dst_latest_data_db = v07.open_db(dst_env, v07.DbType.LATEST_DATA)
+    dst_latest_type_db = v07.open_db(dst_env, v07.DbType.LATEST_TYPE)
+
+    latest_event = None
+    next_event_type_ref = itertools.count(1)
+
+    with src_env.begin(db=src_latest_db, buffers=True) as src_txn:
+        for _, src_encoded_value in src_txn.cursor():
+            src_event = v06.decode_event(src_encoded_value)
+            dst_event = _convert_event(src_event, server_id)
+
+            if not latest_event or latest_event.event_id < dst_event.event_id:
+                latest_event = dst_event
+
+            event_type_ref = next(next_event_type_ref)
+
+            with dst_env.begin(db=dst_latest_data_db, write=True) as dst_txn:
+                dst_txn.put(
+                    v07.encode_latest_data_db_key(
+                        event_type_ref),
+                    v07.encode_latest_data_db_value(
+                        dst_event))
+
+            with dst_env.begin(db=dst_latest_type_db, write=True) as dst_txn:
+                dst_txn.put(
+                    v07.encode_latest_type_db_key(
+                        event_type_ref),
+                    v07.encode_latest_type_db_value(
+                        dst_event.event_type))
+
+            _update_ref_db(dst_env=dst_env,
+                           dst_ref_db=dst_ref_db,
+                           dst_event=dst_event,
+                           event_ref=v07.LatestEventRef(event_type_ref))
+
+    if latest_event:
+        _update_system_db(dst_env=dst_env,
+                          dst_system_db=dst_system_db,
+                          server_id=server_id,
+                          latest_event=latest_event)
+
+
+def _convert_ordered(src_env, dst_env, dst_system_db, dst_ref_db, server_id):
+    src_ordered_data_db = src_env.open_db(b'ordered_data')
+    src_ordered_partition_db = src_env.open_db(b'ordered_partition')
+    src_ordered_count_db = src_env.open_db(b'ordered_count')
+
+    dst_ordered_data_db = v07.open_db(dst_env, v07.DbType.ORDERED_DATA)
+    dst_ordered_partition_db = v07.open_db(
+        dst_env, v07.DbType.ORDERED_PARTITION)
+    dst_ordered_count_db = v07.open_db(dst_env, v07.DbType.ORDERED_COUNT)
+
+    latest_event = None
+
+    with src_env.begin(db=src_ordered_data_db, buffers=True) as src_txn:
+        for src_encoded_key, src_encoded_value in src_txn.cursor():
+            partition_id, src_timestamp, _ = v06.decode_uint_timestamp_uint(
+                src_encoded_key)
+            src_event = v06.decode_event(src_encoded_value)
+            dst_timestamp = _convert_timestamp(src_timestamp)
+            dst_event = _convert_event(src_event, server_id)
+
+            if not latest_event or latest_event.event_id < dst_event.event_id:
+                latest_event = dst_event
+
+            dst_key = partition_id, dst_timestamp, dst_event.event_id
+
+            with dst_env.begin(db=dst_ordered_data_db, write=True) as dst_txn:
+                dst_txn.put(
+                    v07.encode_ordered_data_db_key(
+                        dst_key),
+                    v07.encode_ordered_data_db_value(
+                        dst_event))
+
+            _update_ref_db(dst_env=dst_env,
+                           dst_ref_db=dst_ref_db,
+                           dst_event=dst_event,
+                           event_ref=v07.OrderedEventRef(dst_key))
+
+    if latest_event:
+        _update_system_db(dst_env=dst_env,
+                          dst_system_db=dst_system_db,
+                          server_id=server_id,
+                          latest_event=latest_event)
+
+    with src_env.begin(db=src_ordered_partition_db, buffers=True) as src_txn:
+        for src_encoded_key, src_encoded_value in src_txn.cursor():
+            partition_id = v06.decode_uint(src_encoded_key)
+            partition_data = v06.decode_json(src_encoded_value)
+
+            with dst_env.begin(db=dst_ordered_partition_db,
+                               write=True) as dst_txn:
+                dst_txn.put(
+                    v07.encode_ordered_partition_db_key(
+                        partition_id),
+                    v07.encode_ordered_partition_db_value(
+                        partition_data))
+
+    with src_env.begin(db=src_ordered_count_db, buffers=True) as src_txn:
+        for src_encoded_key, src_encoded_value in src_txn.cursor():
+            partition_id = v06.decode_uint(src_encoded_key)
+            count = v06.decode_json(src_encoded_value)
+
+            with dst_env.begin(db=dst_ordered_count_db,
+                               write=True) as dst_txn:
+                dst_txn.put(
+                    v07.encode_ordered_count_db_key(
+                        partition_id),
+                    v07.encode_ordered_count_db_value(
+                        count))
+
+
+def _update_ref_db(dst_env, dst_ref_db, dst_event, event_ref):
+    with dst_env.begin(db=dst_ref_db,
+                       write=True,
+                       buffers=True) as dst_txn:
+        dst_encoded_key = v07.encode_ref_db_key(dst_event.event_id)
+        dst_encoded_value = dst_txn.get(dst_encoded_key)
+
+        event_refs = (v07.decode_ref_db_value(dst_encoded_value)
+                      if dst_encoded_value else set())
+        event_refs.add(event_ref)
+        dst_encoded_value = v07.encode_ref_db_value(event_refs)
+
+        dst_txn.put(dst_encoded_key, dst_encoded_value)
+
+
+def _update_system_db(dst_env, dst_system_db, server_id, latest_event):
+    with dst_env.begin(db=dst_system_db,
+                       write=True,
+                       buffers=True) as dst_txn:
+        dst_encoded_key = v07.encode_system_db_key(server_id)
+        dst_encoded_value = dst_txn.get(dst_encoded_key)
+
+        event_id, _ = (v07.decode_system_db_value(dst_encoded_value)
+                       if dst_encoded_value
+                       else (v07.EventId(server_id, 0, 0), None))
+
+        if latest_event.event_id > event_id:
+            dst_txn.put(dst_encoded_key,
+                        v07.encode_system_db_value((latest_event.event_id,
+                                                    latest_event.timestamp)))
+
+
+def _get_server_id(src_env, src_system_db):
+    with src_env.begin(db=src_system_db, buffers=True) as txn:
+        src_encoded_server_id = txn.get(b'server_id')
+        return v06.decode_uint(src_encoded_server_id)
+
+
+def _convert_event(src_event, server_id):
+    return v07.Event(
+        event_id=_convert_event_id(src_event.event_id, server_id),
+        event_type=src_event.event_type,
+        timestamp=_convert_timestamp(src_event.timestamp),
+        source_timestamp=(_convert_timestamp(src_event.source_timestamp)
+                          if src_event.source_timestamp else None),
+        payload=(_convert_event_payload(src_event.payload)
+                 if src_event.payload else None))
+
+
+def _convert_event_id(src_event_id, server_id):
+    return v07.EventId(server=(src_event_id.server if src_event_id.server > 0
+                               else server_id),
+                       session=src_event_id.instance,
+                       instance=src_event_id.instance)
+
+
+def _convert_event_payload(src_event_payload):
+    return v07.EventPayload(
+        type=v07.EventPayloadType[src_event_payload.type.name],
+        data=src_event_payload.data)
+
+
+def _convert_timestamp(src_timestamp):
+    return v07.Timestamp(s=src_timestamp.s,
+                         us=src_timestamp.us)