ramses-rf 0.53.2__py3-none-any.whl → 0.53.4__py3-none-any.whl

ramses_cli/client.py

@@ -471,13 +471,8 @@ def print_results(gwy: Gateway, **kwargs: Any) -> None:
         system_id, _ = kwargs[GET_SCHED]
 
 
-def _save_state(gwy: Gateway) -> None:
-    """Save the gateway state to files.
-
-    :param gwy: The gateway instance.
-    """
-    schema, msgs = gwy.get_state()
-
+def _write_state(schema: dict[str, Any], msgs: dict[str, str]) -> None:
+    """Write the state to the file system (blocking)."""
     with open("state_msgs.log", "w") as f:
         [f.write(f"{dtm} {pkt}\r\n") for dtm, pkt in msgs.items()]  # if not m._expired
 
@@ -485,13 +480,22 @@ def _save_state(gwy: Gateway) -> None:
         f.write(json.dumps(schema, indent=4))
 
 
-def _print_engine_state(gwy: Gateway, **kwargs: Any) -> None:
+async def _save_state(gwy: Gateway) -> None:
+    """Save the gateway state to files.
+
+    :param gwy: The gateway instance.
+    """
+    schema, msgs = await gwy.get_state()
+    await asyncio.to_thread(_write_state, schema, msgs)
+
+
+async def _print_engine_state(gwy: Gateway, **kwargs: Any) -> None:
     """Print the current engine state (schema and packets).
 
     :param gwy: The gateway instance.
     :param kwargs: Command arguments to determine verbosity.
     """
-    (schema, packets) = gwy.get_state(include_expired=True)
+    (schema, packets) = await gwy.get_state(include_expired=True)
 
     if kwargs["print_state"] > 0:
         print(f"schema: {json.dumps(schema, indent=4)}\r\n")
@@ -671,10 +675,10 @@ async def async_main(command: str, lib_kwargs: dict[str, Any], **kwargs: Any) ->
     print(f"\r\nclient.py: Engine stopped: {msg}")
 
     # if kwargs["save_state"]:
-    #    _save_state(gwy)
+    #    await _save_state(gwy)
 
     if kwargs["print_state"]:
-        _print_engine_state(gwy, **kwargs)
+        await _print_engine_state(gwy, **kwargs)
 
     elif command == EXECUTE:
         print_results(gwy, **kwargs)

ramses_rf/database.py

@@ -24,14 +24,19 @@ RAMSES RF - Message database and index.
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import logging
+import os
 import sqlite3
+import uuid
 from collections import OrderedDict
 from datetime import datetime as dt, timedelta as td
 from typing import TYPE_CHECKING, Any, NewType
 
 from ramses_tx import CODES_SCHEMA, RQ, Code, Message, Packet
 
+from .storage import StorageWorker
+
 if TYPE_CHECKING:
     DtmStrT = NewType("DtmStrT", str)
     MsgDdT = OrderedDict[DtmStrT, Message]
@@ -89,22 +94,55 @@ class MessageIndex:
 
     _housekeeping_task: asyncio.Task[None]
 
-    def __init__(self, maintain: bool = True) -> None:
+    def __init__(self, maintain: bool = True, db_path: str = ":memory:") -> None:
         """Instantiate a message database/index."""
 
         self.maintain = maintain
         self._msgs: MsgDdT = OrderedDict()  # stores all messages for retrieval.
         # Filled & cleaned up in housekeeping_loop.
 
-        # Connect to a SQLite DB in memory
+        # For :memory: databases with multiple connections (Reader vs Worker)
+        # We must use a Shared Cache URI so both threads see the same data.
+        if db_path == ":memory:":
+            # Unique ID ensures parallel tests don't share the same in-memory DB
+            db_path = f"file:ramses_rf_{uuid.uuid4()}?mode=memory&cache=shared"
+
+        # Start the Storage Worker (Write Connection)
+        # This thread handles all blocking INSERT/UPDATE operations
+        self._worker = StorageWorker(db_path)
+
+        # Wait for the worker to create the tables.
+        # This prevents "no such table" errors on immediate reads.
+        if not self._worker.wait_for_ready(timeout=10.0):
+            _LOGGER.error("MessageIndex: StorageWorker timed out initializing database")
+
+        # Connect to a SQLite DB (Read Connection)
         self._cx = sqlite3.connect(
-            ":memory:", detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES
+            db_path,
+            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
+            check_same_thread=False,
+            uri=True,  # Enable URI parsing for shared memory support
+            timeout=10.0,  # Increased timeout to reduce 'database locked' errors
+            isolation_level=None,  # Autocommit mode prevents stale snapshots
         )
+
+        # Enable Write-Ahead Logging for Reader as well
+        if db_path != ":memory:" and "mode=memory" not in db_path:
+            with contextlib.suppress(sqlite3.Error):
+                self._cx.execute("PRAGMA journal_mode=WAL")
+        elif "cache=shared" in db_path:
+            # Shared cache (used in tests) requires read_uncommitted to prevent
+            # readers from blocking writers (Table Locking).
+            with contextlib.suppress(sqlite3.Error):
+                self._cx.execute("PRAGMA read_uncommitted = true")
+
         # detect_types should retain dt type on store/retrieve
         self._cu = self._cx.cursor()  # Create a cursor
 
         _setup_db_adapters()  # DTM adapter/converter
-        self._setup_db_schema()
+
+        # Schema creation is now handled safely by the StorageWorker to avoid races.
+        # self._setup_db_schema()
 
         if self.maintain:
             self._lock = asyncio.Lock()
@@ -137,6 +175,7 @@ class MessageIndex:
         ):
             self._housekeeping_task.cancel()  # stop the housekeeper
 
+        self._worker.stop()  # Stop the background thread
         self._cx.commit()  # just in case
         self._cx.close()  # may still need to do queries after engine has stopped?
 
@@ -145,6 +184,13 @@ class MessageIndex:
         """Return the messages in the index in a threadsafe way."""
         return self._msgs
 
+    def flush(self) -> None:
+        """Flush the storage worker queue.
+
+        This is primarily for testing to ensure data persistence before querying.
+        """
+        self._worker.flush()
+
     def _setup_db_schema(self) -> None:
         """Set up the message database schema.
 
@@ -236,23 +282,30 @@ class MessageIndex:
         dup: tuple[Message, ...] = tuple()  # avoid UnboundLocalError
         old: Message | None = None  # avoid UnboundLocalError
 
+        # Check in-memory cache for collision instead of blocking SQL
+        dtm_str: DtmStrT = msg.dtm.isoformat(timespec="microseconds")  # type: ignore[assignment]
+        if dtm_str in self._msgs:
+            dup = (self._msgs[dtm_str],)
+
         try:  # TODO: remove this, or apply only when source is a real packet log?
             # await self._lock.acquire()
-            dup = self._delete_from(  # HACK: because of contrived pkt logs
-                dtm=msg.dtm  # stored as such with DTM formatter
-            )
-            old = self._insert_into(msg)  # will delete old msg by hdr (not dtm!)
+            # dup = self._delete_from(  # HACK: because of contrived pkt logs
+            #     dtm=msg.dtm  # stored as such with DTM formatter
+            # )
+            # We defer the write to the worker; return value (old) is not available synchronously
+            self._insert_into(msg)  # will delete old msg by hdr (not dtm!)
 
         except (
             sqlite3.Error
         ):  # UNIQUE constraint failed: ? messages.dtm or .hdr (so: HACK)
-            self._cx.rollback()
+            # self._cx.rollback()
+            pass
 
         else:
             # _msgs dict requires a timestamp reformat
-            dtm: DtmStrT = msg.dtm.isoformat(timespec="microseconds")  # type: ignore[assignment]
+            # dtm: DtmStrT = msg.dtm.isoformat(timespec="microseconds")
             # add msg to self._msgs dict
-            self._msgs[dtm] = msg
+            self._msgs[dtm_str] = msg
 
         finally:
             pass  # self._lock.release()
@@ -288,39 +341,36 @@ class MessageIndex:
         dtm: DtmStrT = _now.isoformat(timespec="microseconds")  # type: ignore[assignment]
         hdr = f"{code}|{verb}|{src}|{payload}"
 
-        dup = self._delete_from(hdr=hdr)
+        # dup = self._delete_from(hdr=hdr)
+        # Avoid blocking read; worker handles REPLACE on unique constraint collision
+
+        # Prepare data tuple for worker
+        data = (
+            _now,
+            verb,
+            src,
+            src,
+            code,
+            None,
+            hdr,
+            "|",
+        )
 
-        sql = """
-            INSERT INTO messages (dtm, verb, src, dst, code, ctx, hdr, plk)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-        """
-        try:
-            self._cu.execute(
-                sql,
-                (
-                    _now,
-                    verb,
-                    src,
-                    src,
-                    code,
-                    None,
-                    hdr,
-                    "|",
-                ),
-            )
-        except sqlite3.Error:
-            self._cx.rollback()
-        else:
-            # also add dummy 3220 msg to self._msgs dict to allow maintenance loop
-            msg: Message = Message._from_pkt(
-                Packet(
-                    _now, f"... {verb} --- {src} --:------ {src} {code} 005 0000000000"
-                )
-            )
-            self._msgs[dtm] = msg
+        self._worker.submit_packet(data)
+
+        # Backward compatibility for Tests:
+        # Check specific env var set by pytest, which is more reliable than sys.modules
+        if "PYTEST_CURRENT_TEST" in os.environ:
+            self.flush()
+
+        # also add dummy 3220 msg to self._msgs dict to allow maintenance loop
+        msg: Message = Message._from_pkt(
+            Packet(_now, f"... {verb} --- {src} --:------ {src} {code} 005 0000000000")
+        )
+        self._msgs[dtm] = msg
 
-        if dup:  # expected when more than one heat system in schema
-            _LOGGER.debug("Replaced record with same hdr: %s", hdr)
+        # if dup:  # expected when more than one heat system in schema
+        #     _LOGGER.debug("Replaced record with same hdr: %s", hdr)
 
     def _insert_into(self, msg: Message) -> Message | None:
         """
@@ -337,29 +387,31 @@ class MessageIndex:
         else:
             msg_pkt_ctx = msg._pkt._ctx  # can be None
 
-        _old_msgs = self._delete_from(hdr=msg._pkt._hdr)
+        # _old_msgs = self._delete_from(hdr=msg._pkt._hdr)
+        # Refactor: Worker uses INSERT OR REPLACE to handle collision
+
+        data = (
+            msg.dtm,
+            str(msg.verb),
+            msg.src.id,
+            msg.dst.id,
+            str(msg.code),
+            msg_pkt_ctx,
+            msg._pkt._hdr,
+            payload_keys(msg.payload),
+        )
 
-        sql = """
-            INSERT INTO messages (dtm, verb, src, dst, code, ctx, hdr, plk)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-        """
+        self._worker.submit_packet(data)
+
+        # Backward compatibility for Tests:
+        # Tests assume the DB update is instant. If running in pytest, flush immediately.
+        # This effectively makes the operation synchronous during tests to avoid rewriting tests.
+        if "PYTEST_CURRENT_TEST" in os.environ:
+            self.flush()
 
-        self._cu.execute(
-            sql,
-            (
-                msg.dtm,
-                str(msg.verb),
-                msg.src.id,
-                msg.dst.id,
-                str(msg.code),
-                msg_pkt_ctx,
-                msg._pkt._hdr,
-                payload_keys(msg.payload),
-            ),
-        )
         # _LOGGER.debug(f"Added {msg} to gwy.msg_db")
 
-        return _old_msgs[0] if _old_msgs else None
+        return None
 
     def rem(
         self, msg: Message | None = None, **kwargs: str | dt

ramses_rf/gateway.py

@@ -267,12 +267,14 @@ class Gateway(Engine):
         :returns: None
         :rtype: None
         """
+        # Stop the Engine first to ensure no tasks/callbacks try to write
+        # to the DB while we are closing it.
+        await super().stop()
 
         if self.msg_db:
             self.msg_db.stop()
-        await super().stop()
 
-    def _pause(self, *args: Any) -> None:
+    async def _pause(self, *args: Any) -> None:
         """Pause the (unpaused) gateway (disables sending/discovery).
 
         There is the option to save other objects, as `args`.
@@ -288,12 +290,12 @@ class Gateway(Engine):
         self.config.disable_discovery, disc_flag = True, self.config.disable_discovery
 
         try:
-            super()._pause(disc_flag, *args)
+            await super()._pause(disc_flag, *args)
         except RuntimeError:
             self.config.disable_discovery = disc_flag
             raise
 
-    def _resume(self) -> tuple[Any]:
+    async def _resume(self) -> tuple[Any]:
         """Resume the (paused) gateway (enables sending/discovery, if applicable).
 
         Will restore other objects, as `args`.
@@ -305,11 +307,13 @@ class Gateway(Engine):
 
         _LOGGER.debug("Gateway: Resuming engine...")
 
-        self.config.disable_discovery, *args = super()._resume()  # type: ignore[assignment]
+        # args_tuple = await super()._resume()
+        # self.config.disable_discovery, *args = args_tuple  # type: ignore[assignment]
+        self.config.disable_discovery, *args = await super()._resume()  # type: ignore[assignment]
 
         return args
 
-    def get_state(
+    async def get_state(
         self, include_expired: bool = False
     ) -> tuple[dict[str, Any], dict[str, str]]:
         """Return the current schema & state (may include expired packets).
@@ -320,7 +324,7 @@ class Gateway(Engine):
         :rtype: tuple[dict[str, Any], dict[str, str]]
         """
 
-        self._pause()
+        await self._pause()
 
         def wanted_msg(msg: Message, include_expired: bool = False) -> bool:
             if msg.code == Code._313F:
@@ -357,7 +361,7 @@ class Gateway(Engine):
             }
             # _LOGGER.warning("Missing MessageIndex")
 
-        self._resume()
+        await self._resume()
 
         return self.schema, dict(sorted(pkts.items()))
 
@@ -392,7 +396,7 @@ class Gateway(Engine):
         tmp_transport: RamsesTransportT  # mypy hint
 
         _LOGGER.debug("Gateway: Restoring a cached packet log...")
-        self._pause()
+        await self._pause()
 
         if _clear_state:  # only intended for test suite use
             clear_state()
@@ -428,7 +432,7 @@ class Gateway(Engine):
         await tmp_transport.get_extra_info(SZ_READER_TASK)
 
         _LOGGER.debug("Gateway: Restored, resuming")
-        self._resume()
+        await self._resume()
 
     def _add_device(self, dev: Device) -> None:  # TODO: also: _add_system()
         """Add a device to the gateway (called by devices during instantiation).

ramses_rf/storage.py

@@ -0,0 +1,168 @@
+"""RAMSES RF - Background storage worker for async I/O."""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import queue
+import sqlite3
+import threading
+from typing import Any
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class StorageWorker:
+    """A background worker thread to handle blocking storage I/O asynchronously."""
+
+    def __init__(self, db_path: str = ":memory:"):
+        """Initialize the storage worker thread."""
+        self._db_path = db_path
+        self._queue: queue.SimpleQueue[tuple[str, Any] | None] = queue.SimpleQueue()
+        self._ready_event = threading.Event()
+
+        self._thread = threading.Thread(
+            target=self._run,
+            name="RamsesStorage",
+            daemon=True,  # FIX: Set to True so the process can exit even if stop() is missed
+        )
+        self._thread.start()
+
+    def wait_for_ready(self, timeout: float | None = None) -> bool:
+        """Wait until the database is initialized and ready."""
+        return self._ready_event.wait(timeout)
+
+    def submit_packet(self, packet_data: tuple[Any, ...]) -> None:
+        """Submit a packet tuple for SQL insertion (Non-blocking)."""
+        self._queue.put(("SQL", packet_data))
+
+    def flush(self, timeout: float = 10.0) -> None:
+        """Block until all currently pending tasks are processed."""
+        # REMOVED: if self._queue.empty(): return
+        # This check caused a race condition where flush() returned before
+        # the worker finished committing the last item it just popped.
+
+        # We inject a special marker into the queue
+        sentinel = threading.Event()
+        self._queue.put(("MARKER", sentinel))
+
+        # Wait for the worker to set the sentinel
+        if not sentinel.wait(timeout):
+            _LOGGER.warning("StorageWorker flush timed out")
+
+    def stop(self) -> None:
+        """Signal the worker to stop processing and close resources."""
+        self._queue.put(None)  # Poison pill
+        self._thread.join()
+
+    def _init_db(self, conn: sqlite3.Connection) -> None:
+        """Initialize the database schema."""
+        cursor = conn.cursor()
+        cursor.execute(
+            """
+            CREATE TABLE IF NOT EXISTS messages (
+                dtm    DTM      NOT NULL PRIMARY KEY,
+                verb   TEXT(2)  NOT NULL,
+                src    TEXT(12) NOT NULL,
+                dst    TEXT(12) NOT NULL,
+                code   TEXT(4)  NOT NULL,
+                ctx    TEXT,
+                hdr    TEXT     NOT NULL UNIQUE,
+                plk    TEXT     NOT NULL
+            )
+            """
+        )
+        # Create indexes to speed up future reads
+        for col in ("verb", "src", "dst", "code", "ctx", "hdr"):
+            cursor.execute(f"CREATE INDEX IF NOT EXISTS idx_{col} ON messages ({col})")
+        conn.commit()
+
+    def _run(self) -> None:
+        """The main loop running in the background thread."""
+        _LOGGER.debug("StorageWorker thread started.")
+
+        # Setup SQLite connection in this thread
+        try:
+            # uri=True allows opening "file::memory:?cache=shared"
+            conn = sqlite3.connect(
+                self._db_path,
+                detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
+                check_same_thread=False,
+                uri=True,
+                timeout=10.0,  # Increased timeout for locking
+            )
+
+            # Enable Write-Ahead Logging for concurrency
+            if self._db_path != ":memory:" and "mode=memory" not in self._db_path:
+                with contextlib.suppress(sqlite3.Error):
+                    conn.execute("PRAGMA journal_mode=WAL")
+                    conn.execute("PRAGMA synchronous=NORMAL")
+            elif "cache=shared" in self._db_path:
+                with contextlib.suppress(sqlite3.Error):
+                    conn.execute("PRAGMA read_uncommitted = true")
+
+            self._init_db(conn)
+            self._ready_event.set()  # Signal that tables exist
+        except sqlite3.Error as exc:
+            _LOGGER.error("Failed to initialize storage database: %s", exc)
+            self._ready_event.set()  # Avoid blocking waiters forever
+            return
+
+        while True:
+            try:
+                # Block here waiting for work
+                item = self._queue.get()
+
+                if item is None:  # Shutdown signal
+                    break
+
+                task_type, data = item
+
+                if task_type == "MARKER":
+                    # Flush requested
+                    data.set()
+                    continue
+
+                if task_type == "SQL":
+                    # Optimization: Batch processing
+                    batch = [data]
+                    # Drain queue of pending SQL tasks to bulk insert
+                    while not self._queue.empty():
+                        try:
+                            # Peek/get next item without blocking
+                            next_item = self._queue.get_nowait()
+                            if next_item is None:
+                                self._queue.put(None)  # Re-queue poison pill
+                                break
+
+                            next_type, next_data = next_item
+                            if next_type == "SQL":
+                                batch.append(next_data)
+                            elif next_type == "MARKER":
+                                # Handle marker after this batch
+                                self._queue.put(next_item)  # Re-queue marker
+                                break
+                            else:
+                                pass
+                        except queue.Empty:
+                            break
+
+                    try:
+                        conn.executemany(
+                            """
+                            INSERT OR REPLACE INTO messages 
+                            (dtm, verb, src, dst, code, ctx, hdr, plk)
+                            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                            """,
+                            batch,
+                        )
+                        conn.commit()
+                    except sqlite3.Error as err:
+                        _LOGGER.error("SQL Write Failed: %s", err)
+
+            except Exception as err:
+                _LOGGER.exception("StorageWorker encountered an error: %s", err)
+
+        # Cleanup
+        conn.close()
+        _LOGGER.debug("StorageWorker thread stopped.")

ramses_rf/version.py

@@ -1,4 +1,4 @@
 """RAMSES RF - a RAMSES-II protocol decoder & analyser (application layer)."""
 
-__version__ = "0.53.2"
+__version__ = "0.53.4"
 VERSION = __version__

{ramses_rf-0.53.2.dist-info → ramses_rf-0.53.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ramses_rf
-Version: 0.53.2
+Version: 0.53.4
 Summary: A stateful RAMSES-II protocol decoder & analyser.
 Project-URL: Homepage, https://github.com/ramses-rf/ramses_rf
 Project-URL: Bug Tracker, https://github.com/ramses-rf/ramses_rf/issues

{ramses_rf-0.53.2.dist-info → ramses_rf-0.53.4.dist-info}/RECORD

@@ -1,5 +1,5 @@
 ramses_cli/__init__.py,sha256=d_3uIFkK8JnWOxknrBloKCe6-vI9Ouo_KGqR4kfBQW8,417
-ramses_cli/client.py,sha256=Fd6hXGBvI-i89Xu8QbBU9NW-dWlQjSer-_LYAtvJPW4,25055
+ramses_cli/client.py,sha256=w95Xv2_kVlYelI5XnGt6D2QVLG3guiSMqo_MO1Ni-dc,25277
 ramses_cli/debug.py,sha256=PLcz-3PjUiMVqtD_p6VqTA92eHUM58lOBFXh_qgQ_wA,576
 ramses_cli/discovery.py,sha256=WTcoFH5hNhQ1AeOZtpdZIVYwdUfmUKlq2iBpa-KcgoI,12512
 ramses_cli/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -8,15 +8,16 @@ ramses_cli/utils/convert.py,sha256=N3LxGe3_0pclijtmYW-ChqCuPTzbkoJA4XNAnoSnBk0,1
 ramses_rf/__init__.py,sha256=AXsCK1Eh9FWeAI9D_zY_2KB0dqrTb9a5TNY1NvyQaDM,1271
 ramses_rf/binding_fsm.py,sha256=fuqvcc9YW-wr8SPH8zadpPqrHAvzl_eeWF-IBtlLppY,26632
 ramses_rf/const.py,sha256=L3z31CZ-xqno6oZp_h-67CB_5tDDqTwSWXsqRtsjMcs,5460
-ramses_rf/database.py,sha256=AHXIx5pO0qgiHbBQxLBegnk0e1M4SFwD9Spd4eyW_eQ,21428
+ramses_rf/database.py,sha256=Fv3Xv6S_7qOf6-biHHKZvntkB8ps_SJvjPlKX0pzGfg,23919
 ramses_rf/dispatcher.py,sha256=YjEU-QrBLo9IfoEhJo2ikg_FxOaMYoWvzelr9Vi-JZ8,11398
 ramses_rf/entity_base.py,sha256=L47P_6CRz3tLDzOzII9AgmueKDb-Bp7Ot3vVsr8jo10,59121
 ramses_rf/exceptions.py,sha256=mt_T7irqHSDKir6KLaf6oDglUIdrw0S40JbOrWJk5jc,3657
-ramses_rf/gateway.py,sha256=hHsXbpeZ2Fetpu3jzv8UeUwlNf4rsxOD7eOHuLY83Xk,30101
+ramses_rf/gateway.py,sha256=3rnKm-OunN-O_T0eS9ZayEvg49WMVe7GF4pJhyGx3Io,30409
 ramses_rf/helpers.py,sha256=TNk_QkpIOB3alOp1sqnA9LOzi4fuDCeapNlW3zTzNas,4250
 ramses_rf/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ramses_rf/schemas.py,sha256=X1GAK3kttuLMiSCUDY2s-85fgBxPeU8xiDa6gJ1I5mY,13543
-ramses_rf/version.py,sha256=zQdCsfKjSv1T1jLD9vTQU017G5KsLBSifdP9kV1wzQs,125
+ramses_rf/storage.py,sha256=lGKUgQzXBUwksmEeHMLoVoKPCMPAWWuzwCXM8G2CKmg,6452
+ramses_rf/version.py,sha256=4QvpG_-UTxyow-YNsbbOguS-J5cqoyPKHelB0WfKMkA,125
 ramses_rf/device/__init__.py,sha256=sUbH5dhbYFXSoM_TPFRutpRutBRpup7_cQ9smPtDTy8,4858
 ramses_rf/device/base.py,sha256=Tu5I8Lj7KplfRsIBQAYjilS6YPgTyjpU8qgKugMR2Jk,18281
 ramses_rf/device/heat.py,sha256=CU6GlIgjuYD21braJ_RJlS56zP47TGXNxXnZeavfEMY,54654
@@ -33,24 +34,24 @@ ramses_tx/const.py,sha256=jiE2UaGBJ5agr68EMrcEHWtVz2KMidU7c7rRYCIiaoM,33010
 ramses_tx/exceptions.py,sha256=FJSU9YkvpKjs3yeTqUJX1o3TPFSe_B01gRGIh9b3PNc,2632
 ramses_tx/fingerprints.py,sha256=nfftA1E62HQnb-eLt2EqjEi_la0DAoT0wt-PtTMie0s,11974
 ramses_tx/frame.py,sha256=GzNsXr15YLeidJYGtk_xPqsZQh4ehDDlUCtT6rTDhT8,22046
-ramses_tx/gateway.py,sha256=H4JtIkp7JFMyZZ76sij67rTbjc1i3iWaIf6tuxIpHAg,11529
+ramses_tx/gateway.py,sha256=kJv3jRI66Ii-kxZO6ItbPODW2oJf-e46ou2JdQt0yTY,11498
 ramses_tx/helpers.py,sha256=96OvSOWYuMcr89_c-3dRnqHZaMOctCO94uo1hETh3bc,33613
 ramses_tx/logger.py,sha256=1iKRHKUaqHqGd76CkE_6mCVR0sYODtxshRRwfY61fTk,10426
 ramses_tx/message.py,sha256=zsyDQztSUYeqj3-P598LSmy9ODQY2BUCzWxSoZds6bM,13953
 ramses_tx/opentherm.py,sha256=58PXz9l5x8Ou6Fm3y-R_UnGHCYahoi2RKIDdYStUMzk,42378
 ramses_tx/packet.py,sha256=_nzuInS_WhdOI26SYvgsdDqIaDvVNguc2YDwdPOVCbU,7661
-ramses_tx/parsers.py,sha256=vHaVdapzW9TtQhCjR9XuaLLkp5ZMXKmTm05zbu0ekW0,148576
+ramses_tx/parsers.py,sha256=CJKdLF1F5KR7MwYxwkwSvrusTONh5FGecj7_eeBWu7A,148609
 ramses_tx/protocol.py,sha256=nBPKCD1tcGp_FiX0qhsY0XoGO_h87w5cYywBjSpum4w,33048
 ramses_tx/protocol_fsm.py,sha256=o9vLvlXor3LkPgsY1zii5P1R01GzYLf_PECDdoxtC24,27520
 ramses_tx/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ramses_tx/ramses.py,sha256=vp748Tf_a-56OMM8CWDA2ZktRfTuj0QVyPRcnsOstSM,53983
+ramses_tx/ramses.py,sha256=V4LqD6IaohU7TTZp-_f1K2SOCJwzRY0v8_-INESh2cU,53986
 ramses_tx/schemas.py,sha256=Hrmf_q9bAZtkKJzGu6GtUO0QV_-K9i4L99EzGWR13eE,13408
-ramses_tx/transport.py,sha256=hktWSDOzKUME967w58JB4z3MgmCMiQ0QZI0x4-Awaas,76419
+ramses_tx/transport.py,sha256=RIrcNrJwiKB_xmJLgG4Z--V2d83PLsJnLXZK-WFFgsA,76568
 ramses_tx/typed_dicts.py,sha256=w-0V5t2Q3GiNUOrRAWiW9GtSwbta_7luME6GfIb1zhI,10869
 ramses_tx/typing.py,sha256=eF2SlPWhNhEFQj6WX2AhTXiyRQVXYnFutiepllYl2rI,5042
-ramses_tx/version.py,sha256=pMxS8r8tPzENEJqDuPgdv_jZrE1H4bKjkrFTvQrqYBs,123
-ramses_rf-0.53.2.dist-info/METADATA,sha256=NScfGCAX7fWvI8OE8H8BxGNFDw9Pd8-kKAehSi_FW1Q,4179
-ramses_rf-0.53.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-ramses_rf-0.53.2.dist-info/entry_points.txt,sha256=NnyK29baOCNg8DinPYiZ368h7MTH7bgTW26z2A1NeIE,50
-ramses_rf-0.53.2.dist-info/licenses/LICENSE,sha256=ptVutrtSMr7X-ek6LduiD8Cce4JsNn_8sR8MYlm-fvo,1086
-ramses_rf-0.53.2.dist-info/RECORD,,
+ramses_tx/version.py,sha256=hutrhdcMJOwR4LO2siGwJHfwUmCNdXAWpBV8-SaqUVo,123
+ramses_rf-0.53.4.dist-info/METADATA,sha256=GNBabuxJgwv1H80fzFeCO31BSn0yI76V0cTeMKp2HjE,4179
+ramses_rf-0.53.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+ramses_rf-0.53.4.dist-info/entry_points.txt,sha256=NnyK29baOCNg8DinPYiZ368h7MTH7bgTW26z2A1NeIE,50
+ramses_rf-0.53.4.dist-info/licenses/LICENSE,sha256=ptVutrtSMr7X-ek6LduiD8Cce4JsNn_8sR8MYlm-fvo,1086
+ramses_rf-0.53.4.dist-info/RECORD,,

ramses_tx/gateway.py

@@ -12,7 +12,6 @@ import asyncio
 import logging
 from collections.abc import Callable
 from datetime import datetime as dt
-from threading import Lock
 from typing import TYPE_CHECKING, Any, Never
 
 from .address import ALL_DEV_ADDR, HGI_DEV_ADDR, NON_DEV_ADDR
@@ -120,7 +119,7 @@ class Engine:
         self._log_all_mqtt = kwargs.pop(SZ_LOG_ALL_MQTT, False)
         self._kwargs: dict[str, Any] = kwargs  # HACK
 
-        self._engine_lock = Lock()  # FIXME: threading lock, or asyncio lock?
+        self._engine_lock = asyncio.Lock()
         self._engine_state: (
             tuple[_MsgHandlerT | None, bool | None, *tuple[Any, ...]] | None
         ) = None
@@ -217,15 +216,13 @@ class Engine:
     async def stop(self) -> None:
         """Close the transport (will stop the protocol)."""
 
-        async def cancel_all_tasks() -> None:  # TODO: needs a lock?
-            _ = [t.cancel() for t in self._tasks if not t.done()]
-            try:  # FIXME: this is broken
-                if tasks := (t for t in self._tasks if not t.done()):
-                    await asyncio.gather(*tasks)
-            except asyncio.CancelledError:
-                pass
+        # Shutdown Safety - wait for tasks to clean up
+        tasks = [t for t in self._tasks if not t.done()]
+        for t in tasks:
+            t.cancel()
 
-        await cancel_all_tasks()
+        if tasks:
+            await asyncio.wait(tasks)
 
         if self._transport:
             self._transport.close()
@@ -233,12 +230,14 @@ class Engine:
 
         return None
 
-    def _pause(self, *args: Any) -> None:
+    async def _pause(self, *args: Any) -> None:
         """Pause the (active) engine or raise a RuntimeError."""
-
-        if not self._engine_lock.acquire(blocking=False):
+        # Async lock handling
+        if self._engine_lock.locked():
             raise RuntimeError("Unable to pause engine, failed to acquire lock")
 
+        await self._engine_lock.acquire()
+
         if self._engine_state is not None:
             self._engine_lock.release()
             raise RuntimeError("Unable to pause engine, it is already paused")
@@ -255,13 +254,18 @@ class Engine:
 
         self._engine_state = (handler, read_only, *args)
 
-    def _resume(self) -> tuple[Any]:  # FIXME: not atomic
+    async def _resume(self) -> tuple[Any]:  # FIXME: not atomic
         """Resume the (paused) engine or raise a RuntimeError."""
 
         args: tuple[Any]  # mypy
 
-        if not self._engine_lock.acquire(timeout=0.1):
-            raise RuntimeError("Unable to resume engine, failed to acquire lock")
+        # Async lock with timeout
+        try:
+            await asyncio.wait_for(self._engine_lock.acquire(), timeout=0.1)
+        except TimeoutError as err:
+            raise RuntimeError(
+                "Unable to resume engine, failed to acquire lock"
+            ) from err
 
         if self._engine_state is None:
             self._engine_lock.release()

ramses_tx/parsers.py

@@ -3109,7 +3109,9 @@ def parser_31da(payload: str, msg: Message) -> PayDictT._31DA:
     if len(payload) == 58:
         return result  # type: ignore[return-value]
 
-    result.update({"_extra": payload[58:]})  # sporadic [58:60] always 00
+    result.update(
+        {"_extra": payload[58:]}
+    )  # sporadic [58:60] one of {00, 20, 40} version?
     return result  # type: ignore[return-value]
 
     # From an Orcon 15RF Display

ramses_tx/ramses.py

@@ -551,7 +551,7 @@ CODES_SCHEMA: dict[Code, dict[str, Any]] = {  # rf_unknown
     },
     Code._31DA: {  # hvac_state (fan_state_extended)
         SZ_NAME: "hvac_state",
-        I_: r"^(00|01|15|16|17|21)[0-9A-F]{56}(00|20)?$",
+        I_: r"^(00|01|15|16|17|21)[0-9A-F]{56}(00|20|40)?$",
         RQ: r"^(00|01|15|16|17|21)$",
         # RQ --- 32:168090 30:082155 --:------ 31DA 001 21
     },

ramses_tx/transport.py

@@ -978,17 +978,21 @@ class FileTransport(_ReadTransport, _FileTransportAbstractor):
         if bool(disable_sending) is False:
             raise exc.TransportSourceInvalid("This Transport cannot send packets")
 
+        self._evt_reading = asyncio.Event()
+
         self._extra[SZ_READER_TASK] = self._reader_task = self._loop.create_task(
             self._start_reader(), name="FileTransport._start_reader()"
         )
 
         self._make_connection(None)
 
-    async def _start_reader(self) -> None:  # TODO
+    async def _start_reader(self) -> None:
         """Start the reader task."""
         self._reading = True
+        self._evt_reading.set()  # Start in reading state
+
         try:
-            await self._reader()
+            await self._producer_loop()
         except Exception as err:
             self.loop.call_soon_threadsafe(
                 functools.partial(self._protocol.connection_lost, err)  # type: ignore[arg-type]
@@ -998,50 +1002,60 @@ class FileTransport(_ReadTransport, _FileTransportAbstractor):
                 functools.partial(self._protocol.connection_lost, None)
             )
 
-    # NOTE: self._frame_read() invoked from here
-    async def _reader(self) -> None:  # TODO
+    def pause_reading(self) -> None:
+        """Pause the receiving end (no data to protocol.pkt_received())."""
+        self._reading = False
+        self._evt_reading.clear()  # Puts the loop to sleep efficiently
+
+    def resume_reading(self) -> None:
+        """Resume the receiving end."""
+        self._reading = True
+        self._evt_reading.set()  # Wakes the loop immediately
+
+    async def _producer_loop(self) -> None:
         """Loop through the packet source for Frames and process them."""
+        # NOTE: fileinput interaction remains synchronous-blocking for simplicity,
+        # but the PAUSE mechanism is now async-non-blocking.
 
         if isinstance(self._pkt_source, dict):
             for dtm_str, pkt_line in self._pkt_source.items():  # assume dtm_str is OK
-                while not self._reading:
-                    await asyncio.sleep(0.001)
-                self._frame_read(dtm_str, pkt_line)
-                await asyncio.sleep(0)
-                # NOTE: instable without, big performance penalty if delay >0
+                await self._process_line(dtm_str, pkt_line)
 
         elif isinstance(self._pkt_source, str):  # file_name, used in client parse
             # open file file_name before reading
             try:
                 with fileinput.input(files=self._pkt_source, encoding="utf-8") as file:
                     for dtm_pkt_line in file:  # self._pkt_source:
-                        # TODO check dtm_str is OK
-                        while not self._reading:
-                            await asyncio.sleep(0.001)
-                        # there may be blank lines in annotated log files
-                        if (dtm_pkt_line := dtm_pkt_line.strip()) and dtm_pkt_line[
-                            :1
-                        ] != "#":
-                            self._frame_read(dtm_pkt_line[:26], dtm_pkt_line[27:])
-                            # this is where the parsing magic happens!
-                        await asyncio.sleep(0)
-                        # NOTE: instable without, big performance penalty if delay >0
+                        await self._process_line_from_raw(dtm_pkt_line)
             except FileNotFoundError as err:
                 _LOGGER.warning(f"Correct the packet file name; {err}")
+
         elif isinstance(self._pkt_source, TextIOWrapper):  # used by client monitor
             for dtm_pkt_line in self._pkt_source:  # should check dtm_str is OK
-                while not self._reading:
-                    await asyncio.sleep(0.001)
-                # can be blank lines in annotated log files
-                if (dtm_pkt_line := dtm_pkt_line.strip()) and dtm_pkt_line[:1] != "#":
-                    self._frame_read(dtm_pkt_line[:26], dtm_pkt_line[27:])
-                await asyncio.sleep(0)
-                # NOTE: instable without, big performance penalty if delay >0
+                await self._process_line_from_raw(dtm_pkt_line)
+
         else:
             raise exc.TransportSourceInvalid(
                 f"Packet source is not dict, TextIOWrapper or str: {self._pkt_source:!r}"
             )
 
+    async def _process_line_from_raw(self, line: str) -> None:
+        """Helper to process raw lines."""
+        # there may be blank lines in annotated log files
+        if (line := line.strip()) and line[:1] != "#":
+            await self._process_line(line[:26], line[27:])
+            # this is where the parsing magic happens!
+
+    async def _process_line(self, dtm_str: str, frame: str) -> None:
+        """Push frame to protocol in a thread-safe way."""
+        # Efficient wait - 0% CPU usage while paused
+        await self._evt_reading.wait()
+
+        self._frame_read(dtm_str, frame)
+
+        # Yield control to the event loop to prevent starvation during large file reads
+        await asyncio.sleep(0)
+
     def _close(self, exc: exc.RamsesException | None = None) -> None:
         """Close the transport (cancel any outstanding tasks).
 

ramses_tx/version.py

@@ -1,4 +1,4 @@
 """RAMSES RF - a RAMSES-II protocol decoder & analyser (transport layer)."""
 
-__version__ = "0.53.2"
+__version__ = "0.53.4"
 VERSION = __version__