hat-event 0.9.27__cp310.cp311.cp312.cp313-abi3-win_amd64.whl

hat/event/backends/lmdb/convert/convert_v07_to_v09.py

@@ -0,0 +1,175 @@
+from pathlib import Path
+
+from hat.event.backends.lmdb.convert import v07
+from hat.event.backends.lmdb.convert import v09
+
+
+def convert(src_path: Path,
+            dst_path: Path):
+    with v07.create_env(src_path) as src_env:
+        with v09.create_env(dst_path) as dst_env:
+            src_dbs = {db_type: v07.open_db(src_env, db_type)
+                       for db_type in v07.DbType}
+            dst_dbs = {db_type: v09.open_db(dst_env, db_type)
+                       for db_type in v09.DbType}
+
+            with src_env.begin(buffers=True) as src_txn:
+                with dst_env.begin(write=True) as dst_txn:
+                    _convert_system_db(src_txn, src_dbs, dst_txn, dst_dbs)
+                    _convert_ref_db(src_txn, src_dbs, dst_txn, dst_dbs)
+                    _convert_latest_db(src_txn, src_dbs, dst_txn, dst_dbs)
+                    _convert_timeseries_db(src_txn, src_dbs, dst_txn, dst_dbs)
+
+
+def _convert_system_db(src_txn, src_dbs, dst_txn, dst_dbs):
+    v09.write(dst_dbs, dst_txn, v09.DbType.SYSTEM_SETTINGS,
+              v09.SettingsId.VERSION, v09.version)
+
+    with src_txn.cursor(db=src_dbs[v07.DbType.SYSTEM]) as src_cursor:
+        for src_key, src_value in src_cursor:
+            server_id = v07.decode_system_db_key(src_key)
+            src_event_id, src_timestamp = v07.decode_system_db_value(src_value)
+
+            dst_event_id = _convert_event_id(src_event_id)
+            dst_timestamp = _convert_timestamp(src_timestamp)
+
+            v09.write(dst_dbs, dst_txn, v09.DbType.SYSTEM_LAST_EVENT_ID,
+                      server_id, dst_event_id)
+
+            v09.write(dst_dbs, dst_txn, v09.DbType.SYSTEM_LAST_TIMESTAMP,
+                      server_id, dst_timestamp)
+
+
+def _convert_ref_db(src_txn, src_dbs, dst_txn, dst_dbs):
+    with src_txn.cursor(db=src_dbs[v07.DbType.REF]) as src_cursor:
+        for src_key, src_value in src_cursor:
+            src_event_id = v07.decode_ref_db_key(src_key)
+            src_event_refs = v07.decode_ref_db_value(src_value)
+
+            dst_event_id = _convert_event_id(src_event_id)
+            dst_event_refs = {_convert_event_ref(src_event_ref)
+                              for src_event_ref in src_event_refs}
+
+            v09.write(dst_dbs, dst_txn, v09.DbType.REF,
+                      dst_event_id, dst_event_refs)
+
+
+def _convert_latest_db(src_txn, src_dbs, dst_txn, dst_dbs):
+    with src_txn.cursor(db=src_dbs[v07.DbType.LATEST_DATA]) as src_cursor:
+        for src_key, src_value in src_cursor:
+            event_type_ref = v07.decode_latest_data_db_key(src_key)
+            src_event = v07.decode_latest_data_db_value(src_value)
+
+            dst_event = _convert_event(src_event)
+
+            v09.write(dst_dbs, dst_txn, v09.DbType.LATEST_DATA,
+                      event_type_ref, dst_event)
+
+    with src_txn.cursor(db=src_dbs[v07.DbType.LATEST_TYPE]) as src_cursor:
+        for src_key, src_value in src_cursor:
+            event_type_ref = v07.decode_latest_type_db_key(src_key)
+            event_type = v07.decode_latest_type_db_value(src_value)
+
+            v09.write(dst_dbs, dst_txn, v09.DbType.LATEST_TYPE,
+                      event_type_ref, event_type)
+
+
+def _convert_timeseries_db(src_txn, src_dbs, dst_txn, dst_dbs):
+    with src_txn.cursor(db=src_dbs[v07.DbType.ORDERED_DATA]) as src_cursor:
+        for src_key, src_value in src_cursor:
+            partition_id, src_timestamp, src_event_id = \
+                v07.decode_ordered_data_db_key(src_key)
+            src_event = v07.decode_ordered_data_db_value(src_value)
+
+            dst_timestamp = _convert_timestamp(src_timestamp)
+            dst_event_id = _convert_event_id(src_event_id)
+            dst_event = _convert_event(src_event)
+
+            v09.write(dst_dbs, dst_txn, v09.DbType.TIMESERIES_DATA,
+                      (partition_id, dst_timestamp, dst_event_id), dst_event)
+
+    with src_txn.cursor(db=src_dbs[v07.DbType.ORDERED_PARTITION]) as src_cursor:  # NOQA
+        for src_key, src_value in src_cursor:
+            partition_id = v07.decode_ordered_partition_db_key(src_key)
+            src_partition_data = v07.decode_ordered_partition_db_value(
+                src_value)
+
+            dst_partition_data = _convert_partition_data(src_partition_data)
+
+            v09.write(dst_dbs, dst_txn, v09.DbType.TIMESERIES_PARTITION,
+                      partition_id, dst_partition_data)
+
+    with src_txn.cursor(db=src_dbs[v07.DbType.ORDERED_COUNT]) as src_cursor:
+        for src_key, src_value in src_cursor:
+            partition_id = v07.decode_ordered_count_db_key(src_key)
+            count = v07.decode_ordered_count_db_value(src_value)
+
+            v09.write(dst_dbs, dst_txn, v09.DbType.TIMESERIES_COUNT,
+                      partition_id, count)
+
+
+def _convert_event_id(src_event_id):
+    return v09.EventId(*src_event_id)
+
+
+def _convert_timestamp(src_timestamp):
+    return v09.Timestamp(*src_timestamp)
+
+
+def _convert_event_ref(src_event_ref):
+    if isinstance(src_event_ref, v07.LatestEventRef):
+        return v09.LatestEventRef(src_event_ref.key)
+
+    if isinstance(src_event_ref, v07.OrderedEventRef):
+        partition_id, src_timestamp, src_event_id = src_event_ref.key
+
+        dst_timestamp = _convert_timestamp(src_timestamp)
+        dst_event_id = _convert_event_id(src_event_id)
+
+        dst_key = partition_id, dst_timestamp, dst_event_id
+        return v09.TimeseriesEventRef(dst_key)
+
+    raise ValueError('unsupported event reference')
+
+
+def _convert_event(src_event):
+    return v09.Event(
+        id=_convert_event_id(src_event.event_id),
+        type=src_event.event_type,
+        timestamp=_convert_timestamp(src_event.timestamp),
+        source_timestamp=(_convert_timestamp(src_event.source_timestamp)
+                          if src_event.source_timestamp else None),
+        payload=_convert_payload(src_event.payload))
+
+
+def _convert_payload(src_payload):
+    if src_payload is None:
+        return
+
+    if src_payload.type == v07.EventPayloadType.BINARY:
+        return v09.EventPayloadBinary(type='',
+                                      data=src_payload.data)
+
+    if src_payload.type == v07.EventPayloadType.JSON:
+        return v09.EventPayloadJson(data=src_payload.data)
+
+    if src_payload.type == v07.EventPayloadType.SBS:
+        binary_type = (f'{src_payload.data.module}.{src_payload.data.type}'
+                       if src_payload.data.module is not None else
+                       src_payload.data.type)
+        return v09.EventPayloadBinary(type=binary_type,
+                                      data=src_payload.data.data)
+
+    raise ValueError('unsupported payload type')
+
+
+def _convert_partition_data(src_data):
+    src_order_by = v07.OrderBy(src_data['order'])
+    dst_order_by = _convert_order_by(src_order_by)
+
+    return {'order': dst_order_by.value,
+            'subscriptions': src_data['subscriptions']}
+
+
+def _convert_order_by(src_order_by):
+    return v09.OrderBy[src_order_by.name]

hat/event/backends/lmdb/convert/main.py

@@ -0,0 +1,88 @@
+from pathlib import Path
+import sys
+import typing
+
+from hat import util
+
+from hat.event.backends.lmdb.convert import convert_v06_to_v07
+from hat.event.backends.lmdb.convert import convert_v07_to_v09
+from hat.event.backends.lmdb.convert.version import Version, get_version
+
+
+class Conversion(typing.NamedTuple):
+    src_version: Version
+    dst_version: Version
+    convert: typing.Callable[[Path, Path], None]
+
+
+target_version = Version.v09
+
+conversions = [Conversion(src_version=Version.v06,
+                          dst_version=Version.v07,
+                          convert=convert_v06_to_v07.convert),
+               Conversion(src_version=Version.v07,
+                          dst_version=Version.v09,
+                          convert=convert_v07_to_v09.convert)]
+
+
+def main():
+    if len(sys.argv) != 3:
+        print(f"Usage: {sys.argv[0]} SRC_DB_PATH DST_DB_PATH", file=sys.stderr)
+        sys.exit(1)
+
+    src_path = Path(sys.argv[1])
+    dst_path = Path(sys.argv[2])
+
+    if not src_path.exists():
+        print(f"invalid SRC_DB_PATH: {src_path}", file=sys.stderr)
+        sys.exit(1)
+
+    if dst_path.exists():
+        print(f"existing DST_DB_PATH: {dst_path}", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        convert(src_path, dst_path)
+
+    except Exception as e:
+        print(f"conversion error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def convert(src_path: Path,
+            dst_path: Path):
+    version = get_version(src_path)
+
+    if version == target_version:
+        raise Exception(f"{src_path} already up to date")
+
+    tmp_src_path = src_path
+
+    while version != target_version:
+        conversion = util.first(conversions,
+                                lambda i: i.src_version == version)
+        if not conversion:
+            raise Exception(f"unsupported version {version.value}")
+
+        if conversion.dst_version == target_version:
+            tmp_dst_path = dst_path
+
+        else:
+            tmp_dst_path = dst_path.with_suffix(
+                f"{dst_path.suffix}.{conversion.dst_version.name}")
+
+        if tmp_dst_path.exists():
+            tmp_dst_path.unlink()
+
+        conversion.convert(tmp_src_path, tmp_dst_path)
+        version = conversion.dst_version
+
+        if tmp_src_path != src_path:
+            tmp_src_path.unlink()
+
+        tmp_src_path = tmp_dst_path
+
+
+if __name__ == '__main__':
+    sys.argv[0] = 'hat-event-lmdb-convert'
+    main()

hat/event/backends/lmdb/convert/v06.py

@@ -0,0 +1,216 @@
+from pathlib import Path
+import enum
+import struct
+import platform
+import typing
+
+import lmdb
+
+from hat import json
+from hat import sbs
+
+
+EventType: typing.TypeAlias = typing.Tuple[str, ...]
+
+
+EventPayloadType = enum.Enum('EventPayloadType', [
+    'BINARY',
+    'JSON',
+    'SBS'])
+
+
+class EventId(typing.NamedTuple):
+    server: int
+    instance: int
+
+
+class SbsData(typing.NamedTuple):
+    module: str | None
+    type: str
+    data: bytes
+
+
+class EventPayload(typing.NamedTuple):
+    type: EventPayloadType
+    data: bytes | json.Data | SbsData
+
+
+class Timestamp(typing.NamedTuple):
+    s: int
+    us: int
+
+
+class Event(typing.NamedTuple):
+    event_id: EventId
+    event_type: EventType
+    timestamp: Timestamp
+    source_timestamp: Timestamp | None
+    payload: EventPayload | None
+
+
+def decode_uint(x: bytes) -> int:
+    return struct.unpack(">Q", x)[0]
+
+
+def decode_timestamp(x: bytes) -> Timestamp:
+    res = struct.unpack(">QI", x)
+    return Timestamp(res[0] - (1 << 63), res[1])
+
+
+def decode_tuple_str(x: bytes) -> typing.Tuple[str, ...]:
+    return tuple(json.decode(str(x, encoding='utf-8')))
+
+
+def decode_json(x: bytes) -> json.Data:
+    return json.decode(str(x, encoding='utf-8'))
+
+
+def decode_uint_timestamp_uint(x: bytes
+                               ) -> tuple[int, Timestamp, int]:
+    res = struct.unpack(">QQIQ", x)
+    return res[0], Timestamp(res[1] - (1 << 63), res[2]), res[3]
+
+
+def decode_event(event_bytes: bytes) -> Event:
+    event_sbs = _sbs_repo.decode('HatEvent.Event', event_bytes)
+    return _event_from_sbs(event_sbs)
+
+
+def create_env(path: Path):
+    max_dbs = 5
+    max_db_size = (512 * 1024 * 1024 * 1024
+                   if platform.architecture()[0] == '64bit'
+                   else 1024 * 1024 * 1024)
+    return lmdb.Environment(str(path),
+                            map_size=max_db_size,
+                            subdir=False,
+                            max_dbs=max_dbs)
+
+
+_sbs_repo = sbs.Repository(r"""
+module HatEvent
+
+MsgSubscribe = Array(EventType)
+
+MsgNotify = Array(Event)
+
+MsgRegisterReq = Array(RegisterEvent)
+
+MsgRegisterRes = Array(Choice {
+    event:    Event
+    failure:  None
+})
+
+MsgQueryReq = QueryData
+
+MsgQueryRes = Array(Event)
+
+Timestamp = Record {
+    s:   Integer
+    us:  Integer
+}
+
+EventId = Record {
+    server:    Integer
+    instance:  Integer
+}
+
+Order = Choice {
+    descending:  None
+    ascending:   None
+}
+
+OrderBy = Choice {
+    timestamp:        None
+    sourceTimestamp:  None
+}
+
+EventType = Array(String)
+
+EventPayload = Choice {
+    binary:  Bytes
+    json:    String
+    sbs:     Record {
+        module:  Optional(String)
+        type:    String
+        data:    Bytes
+    }
+}
+
+Event = Record {
+    id:               EventId
+    type:             EventType
+    timestamp:        Timestamp
+    sourceTimestamp:  Optional(Timestamp)
+    payload:          Optional(EventPayload)
+}
+
+RegisterEvent = Record {
+    type:             EventType
+    sourceTimestamp:  Optional(Timestamp)
+    payload:          Optional(EventPayload)
+}
+
+QueryData = Record {
+    ids:                Optional(Array(EventId))
+    types:              Optional(Array(EventType))
+    tFrom:              Optional(Timestamp)
+    tTo:                Optional(Timestamp)
+    sourceTFrom:        Optional(Timestamp)
+    sourceTTo:          Optional(Timestamp)
+    payload:            Optional(EventPayload)
+    order:              Order
+    orderBy:            OrderBy
+    uniqueType:         Boolean
+    maxResults:         Optional(Integer)
+}
+""")
+
+
+def _event_from_sbs(data: sbs.Data) -> Event:
+    return Event(
+        event_id=_event_id_from_sbs(data['id']),
+        event_type=tuple(data['type']),
+        timestamp=_timestamp_from_sbs(data['timestamp']),
+        source_timestamp=_optional_from_sbs(data['sourceTimestamp'],
+                                            _timestamp_from_sbs),
+        payload=_optional_from_sbs(data['payload'], _event_payload_from_sbs))
+
+
+def _event_payload_from_sbs(data: sbs.Data) -> EventPayload:
+    data_type, data_data = data
+
+    if data_type == 'binary':
+        return EventPayload(type=EventPayloadType.BINARY,
+                            data=data_data)
+
+    if data_type == 'json':
+        return EventPayload(type=EventPayloadType.JSON,
+                            data=json.decode(data_data))
+
+    if data_type == 'sbs':
+        return EventPayload(type=EventPayloadType.SBS,
+                            data=_sbs_data_from_sbs(data_data))
+
+    raise ValueError('unsupported payload type')
+
+
+def _timestamp_from_sbs(data: sbs.Data) -> Timestamp:
+    return Timestamp(s=data['s'], us=data['us'])
+
+
+def _event_id_from_sbs(data: sbs.Data) -> EventId:
+    return EventId(server=data['server'],
+                   instance=data['instance'])
+
+
+def _sbs_data_from_sbs(data: sbs.Data) -> SbsData:
+    return SbsData(module=_optional_from_sbs(data['module']),
+                   type=data['type'],
+                   data=data['data'])
+
+
+def _optional_from_sbs(data: sbs.Data,
+                       fn=lambda i: i
+                       ) -> typing.Any | None:
+    return fn(data[1]) if data[0] == 'value' else None