ramses-rf 0.22.40__py3-none-any.whl → 0.51.1__py3-none-any.whl

ramses_rf/gateway.py

@@ -1,328 +1,85 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-"""RAMSES RF - a RAMSES-II protocol decoder & analyser.
 
-The serial to RF gateway (HGI80, not RFG100).
-"""
+# TODO:
+# - sort out gwy.config...
+# - sort out reduced processing
+
+
+"""RAMSES RF -the gateway (i.e. HGI80 / evofw3, not RFG100)."""
+
 from __future__ import annotations
 
 import asyncio
 import logging
-import os
-import signal
-from concurrent import futures
-from datetime import datetime as dt
-from threading import Lock
 from types import SimpleNamespace
-from typing import Callable, Optional, TextIO
-
-from ramses_rf.device import DeviceHeat, DeviceHvac, Fakeable
-from ramses_rf.protocol.frame import _CodeT, _DeviceIdT, _PayloadT, _VerbT
-from ramses_rf.protocol.protocol import _MessageProtocolT, _MessageTransportT
-from ramses_rf.protocol.transport import _PacketProtocolT, _PacketTransportT
-
-from .const import DONT_CREATE_MESSAGES, SZ_DEVICE_ID, SZ_DEVICES, __dev_mode__
-from .device import Device, device_factory
-from .dispatcher import Message, process_msg
-from .helpers import schedule_task, shrink
-from .protocol import (
-    SZ_POLLER_TASK,
+from typing import TYPE_CHECKING, Any
+
+from ramses_tx import (
     Address,
     Command,
-    create_msg_stack,
-    create_pkt_stack,
+    Engine,
+    Message,
+    Packet,
+    Priority,
+    extract_known_hgi_id,
     is_valid_dev_id,
-    set_logger_timesource,
+    protocol_factory,
     set_pkt_logging_config,
+    transport_factory,
+)
+from ramses_tx.const import (
+    DEFAULT_GAP_DURATION,
+    DEFAULT_MAX_RETRIES,
+    DEFAULT_NUM_REPEATS,
+    DEFAULT_SEND_TIMEOUT,
+    DEFAULT_WAIT_FOR_REPLY,
+    SZ_ACTIVE_HGI,
+)
+from ramses_tx.schemas import (
+    SCH_ENGINE_CONFIG,
+    SZ_BLOCK_LIST,
+    SZ_ENFORCE_KNOWN_LIST,
+    SZ_KNOWN_LIST,
+    PktLogConfigT,
+    PortConfigT,
 )
-from .protocol.address import HGI_DEV_ADDR, NON_DEV_ADDR, NUL_DEV_ADDR
-from .protocol.schemas import SZ_PACKET_LOG, SZ_PORT_CONFIG, SZ_PORT_NAME
+from ramses_tx.transport import SZ_READER_TASK
+
+from .const import DONT_CREATE_MESSAGES, SZ_DEVICES
+from .database import MessageIndex
+from .device import DeviceHeat, DeviceHvac, Fakeable, HgiGateway, device_factory
+from .dispatcher import detect_array_fragment, process_msg
 from .schemas import (
-    SCH_GLOBAL_CONFIG,
+    SCH_GATEWAY_CONFIG,
+    SCH_GLOBAL_SCHEMAS,
     SCH_TRAITS,
     SZ_ALIAS,
-    SZ_BLOCK_LIST,
     SZ_CLASS,
     SZ_CONFIG,
-    SZ_ENFORCE_KNOWN_LIST,
+    SZ_DISABLE_DISCOVERY,
+    SZ_ENABLE_EAVESDROP,
     SZ_FAKED,
-    SZ_KNOWN_LIST,
     SZ_MAIN_TCS,
     SZ_ORPHANS,
-    load_config,
     load_schema,
 )
-from .system import System
+from .system import Evohome
 
-# skipcq: PY-W2000
-from .protocol import (  # noqa: F401, isort: skip, pylint: disable=unused-import
+from .const import (  # noqa: F401, isort: skip, pylint: disable=unused-import
     I_,
     RP,
     RQ,
     W_,
-    F9,
-    FA,
-    FC,
-    FF,
     Code,
 )
 
+if TYPE_CHECKING:
+    from ramses_tx import DeviceIdT, DeviceListT, RamsesTransportT
 
-DEV_MODE = __dev_mode__ and False
+    from .device import Device
+    from .entity_base import Parent
 
 _LOGGER = logging.getLogger(__name__)
-if DEV_MODE:
-    _LOGGER.setLevel(logging.DEBUG)
-
-
-class Engine:
-    """The engine class."""
-
-    _create_msg_stack: Callable = create_msg_stack
-    _create_pkt_stack: Callable = create_pkt_stack
-
-    def __init__(
-        self,
-        port_name: None | str,
-        input_file: None | TextIO = None,
-        port_config: None | dict = None,
-        loop: None | asyncio.AbstractEventLoop = None,
-    ) -> None:
-
-        self.ser_name = port_name
-        self._input_file = input_file
-        self._port_config = port_config or {}
-        self._loop = loop or asyncio.get_running_loop()
-
-        self._include: dict[_DeviceIdT, dict] = {}  # aka known_list, and ?allow_list
-        self._exclude: dict[_DeviceIdT, dict] = {}  # aka block_list
-        self._unwanted: list[_DeviceIdT] = [
-            NON_DEV_ADDR.id,
-            NUL_DEV_ADDR.id,
-            "01:000001",
-        ]
-
-        self.config = SimpleNamespace()  # **SCH_CONFIG_GATEWAY({}))
-
-        self.msg_protocol: _MessageProtocolT = None  # type: ignore[assignment]
-        self.msg_transport: _MessageTransportT = None  # type: ignore[assignment]
-        self.pkt_protocol: _PacketProtocolT = None  # type: ignore[assignment]
-        self.pkt_transport: _PacketTransportT = None  # type: ignore[assignment]
-
-        self._engine_lock = Lock()
-        self._engine_state: None | tuple[None | Callable, tuple] = None
-
-    def __str__(self) -> str:
-        if self.pkt_protocol and self.pkt_protocol._hgi80[SZ_DEVICE_ID]:
-            return f"{self.pkt_protocol._hgi80[SZ_DEVICE_ID]} ({self.ser_name})"
-        return f"{HGI_DEV_ADDR.id} ({self.ser_name})"
-
-    @property
-    def hgi(self) -> None | Device:
-        """Return the active HGI80-compatible gateway device, if known."""
-        if self.pkt_protocol and self.pkt_protocol._hgi80[SZ_DEVICE_ID]:
-            return self.device_by_id.get(self.pkt_protocol._hgi80[SZ_DEVICE_ID])
-        return None
-
-    def _setup_event_handlers(self) -> None:  # HACK: for dev/test only
-        def handle_exception(loop, context):
-            """Handle exceptions on any platform."""
-            _LOGGER.error("handle_exception(): Caught: %s", context["message"])
-
-            exc = context.get("exception")
-            if exc:
-                raise exc
-
-        if DEV_MODE:
-            _LOGGER.debug("_setup_event_handlers(): Creating exception handler...")
-            self._loop.set_exception_handler(handle_exception)
-
-    def _dt_now(self):
-        # return dt.now()
-        return self.pkt_protocol._dt_now() if self.pkt_protocol else dt.now()
-
-    def create_client(
-        self,
-        msg_handler: Callable[[Message, Optional[Message]], None],
-        # msg_filter: Callable[[Message], bool] | None = None,
-    ) -> tuple[_MessageProtocolT, _MessageTransportT]:
-        """Create a client protocol for the RAMSES-II message transport."""
-
-        # The optional filter will return True if the message is to be handled.
-        # """  # TODO
-        # if msg_filter is not None and not is_callback(msg_filter):
-        #     raise TypeError(f"Msg filter {msg_filter} is not a callback")
-        return self._create_msg_stack(msg_handler)
-
-    async def start(self) -> None:
-        self._start()
-
-    def _start(self) -> None:
-        """Initiate ad-hoc sending, and (polled) receiving."""
-
-        (_LOGGER.warning if DEV_MODE else _LOGGER.debug)("ENGINE: Starting poller...")
-
-        pkt_receiver = (
-            self.msg_transport.get_extra_info(self.msg_transport.READER)
-            if self.msg_transport
-            else None
-        )
-
-        if self.ser_name:
-            source = {SZ_PORT_NAME: self.ser_name, SZ_PORT_CONFIG: self._port_config}
-        else:
-            source = {SZ_PACKET_LOG: self._input_file}
-
-        self.pkt_protocol, self.pkt_transport = self._create_pkt_stack(
-            pkt_receiver, **source
-        )  # TODO: may raise SerialException
-
-        if self.ser_name:  # and self.msg_transport:
-            self.msg_transport._set_dispatcher(self.pkt_protocol.send_data)
-        else:  # if self._input_file:
-            set_logger_timesource(self.pkt_protocol._dt_now)
-            _LOGGER.warning("Datetimes maintained as most recent packet log timestamp")
-
-    async def stop(self) -> None:
-        self._stop()
-
-        if (task := self.pkt_source) and not task.done():
-            try:
-                await task
-            except asyncio.CancelledError:
-                pass
-
-    def _stop(self) -> None:
-        """Cancel all outstanding tasks."""
-
-        if self.msg_transport:
-            self.msg_transport.close()  # ? .abort()
-
-        if self.pkt_transport:
-            self.pkt_transport.close()  # ? .abort()
-
-    def _pause(self, *args) -> None:
-        """Pause the (unpaused) engine or raise a RuntimeError."""
-
-        (_LOGGER.info if DEV_MODE else _LOGGER.debug)("ENGINE: Pausing engine...")
-
-        if not self._engine_lock.acquire(blocking=False):
-            raise RuntimeError("Unable to pause engine, failed to acquire lock")
-
-        if self._engine_state is not None:
-            self._engine_lock.release()
-            raise RuntimeError("Unable to pause engine, it is already paused")
-
-        self._engine_state, callback = (None, tuple()), None
-        self._engine_lock.release()
-
-        if self.pkt_protocol:
-            self.pkt_protocol.pause_writing()
-            self.pkt_protocol._callback, callback = None, self.pkt_protocol._callback
-
-        self._engine_state = (callback, args)
-
-    def _resume(self) -> tuple:  # FIXME: not atomic
-        """Resume the (paused) engine or raise a RuntimeError."""
-
-        (_LOGGER.info if DEV_MODE else _LOGGER.debug)("ENGINE: Resuming engine...")
-
-        # if not self.ser_name:
-        #     raise RuntimeError("Unable to resume engine, no serial port configured")
-
-        if not self._engine_lock.acquire(timeout=0.1):
-            raise RuntimeError("Unable to resume engine, failed to acquire lock")
-
-        if self._engine_state is None:
-            self._engine_lock.release()
-            raise RuntimeError("Unable to resume engine, it was not paused")
-
-        callback: None | Callable
-        args: tuple
-        callback, args = self._engine_state
-
-        self._engine_lock.release()
-
-        if self.pkt_protocol:
-            self.pkt_protocol._callback = callback  # self.msg_transport._pkt_receiver
-            self.pkt_protocol.resume_writing()
-
-        self._engine_state = None
-
-        return args
-
-    @property
-    def pkt_source(self) -> None | asyncio.Task:
-        if t := self.msg_transport:
-            return t.get_extra_info(t.WRITER)
-        return None
-
-    @staticmethod
-    def create_cmd(
-        verb: _VerbT, device_id: _DeviceIdT, code: _CodeT, payload: _PayloadT, **kwargs
-    ) -> Command:
-        """Make a command addressed to device_id."""
-        return Command.from_attrs(verb, device_id, code, payload, **kwargs)
-
-    def send_cmd(self, cmd: Command, callback: Callable = None, **kwargs):  # FIXME
-        """Send a command with the option to return any response message via callback.
-
-        Response packets, if any (an RP/I will follow an RQ/W), and have the same code.
-        This routine is thread safe.
-        """
-
-        if not self.msg_protocol:
-            raise RuntimeError("there is no message protocol")
-
-        # self._loop.call_soon_threadsafe(
-        #     self.msg_protocol.send_data(cmd, callback=callback, **kwargs)
-        # )
-        coro = self.msg_protocol.send_data(cmd, callback=callback, **kwargs)
-        fut: futures.Future = asyncio.run_coroutine_threadsafe(coro, self._loop)
-        # fut: asyncio.Future = asyncio.wrap_future(fut)
-        return fut
-
-    async def async_send_cmd(self, cmd: Command, **kwargs) -> None | Message:  # FIXME
-        """Send a command with the option to not wait for a response message.
-
-        Response packets, if any, follow an RQ/W (as an RP/I), and have the same code.
-        This routine is thread safe.
-        """
-
-        # def callback(fut):
-        #     print(fut.result())
-
-        fut = self.send_cmd(cmd, _make_awaitable=True, **kwargs)
-        # fut.add_done_callback(callback)
-
-        while True:
-            try:
-                result = fut.result(timeout=0)
-
-            # except futures.CancelledError:  # fut ?was cancelled by a higher layer
-            #     break
-
-            except futures.TimeoutError:  # fut/cmd has not yet completed
-                pass  # should be a pass
-
-            except TimeoutError as exc:  # raised by send_cmd()
-                raise TimeoutError(f"cmd ({cmd.tx_header}) timed out: {exc}")
-
-            # except RuntimeError as exc:  # raised by send_cmd()
-            #     _LOGGER.error(f"cmd ({cmd.tx_header}) raised an exception: {exc!r}")
-            #     if self.msg_transport.is_closing:
-            #         pass
-
-            except Exception as exc:
-                _LOGGER.error(f"cmd ({cmd.tx_header}) raised an exception: {exc!r}")
-                raise exc
-
-            else:
-                _LOGGER.debug(f"cmd ({cmd.tx_header}) returned: {result!r})")
-                return result
-
-            await asyncio.sleep(0.001)
 
 
 class Gateway(Engine):
@@ -330,81 +87,74 @@ class Gateway(Engine):
 
     def __init__(
         self,
-        port_name: None | str,
-        debug_mode: None | bool = None,
-        input_file: None | TextIO = None,
-        loop: None | asyncio.AbstractEventLoop = None,
-        port_config: None | dict = None,
-        **kwargs,
+        port_name: str | None,
+        input_file: str | None = None,
+        port_config: PortConfigT | None = None,
+        packet_log: PktLogConfigT | None = None,
+        block_list: DeviceListT | None = None,
+        known_list: DeviceListT | None = None,
+        loop: asyncio.AbstractEventLoop | None = None,
+        **kwargs: Any,
     ) -> None:
+        if kwargs.pop("debug_mode", None):
+            _LOGGER.setLevel(logging.DEBUG)
 
-        if debug_mode:
-            _LOGGER.setLevel(logging.DEBUG)  # should be INFO?
-        _LOGGER.debug("Starting RAMSES RF, **config = %s", kwargs)
+        kwargs = {k: v for k, v in kwargs.items() if k[:1] != "_"}  # anachronism
+        config: dict[str, Any] = kwargs.pop(SZ_CONFIG, {})
 
         super().__init__(
-            port_name, input_file=input_file, port_config=port_config, loop=loop
+            port_name,
+            input_file=input_file,
+            port_config=port_config,
+            packet_log=packet_log,
+            block_list=block_list,
+            known_list=known_list,
+            loop=loop,
+            **SCH_ENGINE_CONFIG(config),
         )
 
-        self._tasks: list = []  # TODO: used by discovery, move lower?
-        self._schema: dict[str, dict] = {}
+        if self._disable_sending:
+            config[SZ_DISABLE_DISCOVERY] = True
+        if config.get(SZ_ENABLE_EAVESDROP):
+            _LOGGER.warning(
+                f"{SZ_ENABLE_EAVESDROP}=True: this is strongly discouraged"
+                " for routine use (there be dragons here)"
+            )
 
-        (self.config, self._schema, self._include, self._exclude) = load_config(
-            self.ser_name,
-            self._input_file,
-            **SCH_GLOBAL_CONFIG({k: v for k, v in kwargs.items() if k[:1] != "_"}),
-        )
-        set_pkt_logging_config(
-            cc_console=self.config.reduce_processing >= DONT_CREATE_MESSAGES,
-            **self.config.packet_log or {},
-        )
+        self.config = SimpleNamespace(**SCH_GATEWAY_CONFIG(config))
+        self._schema: dict[str, Any] = SCH_GLOBAL_SCHEMAS(kwargs)
 
-        if self.config.reduce_processing < DONT_CREATE_MESSAGES:
-            self.msg_protocol, self.msg_transport = self.create_client(process_msg)
+        self._tcs: Evohome | None = None
 
-        # if self.config.reduce_processing > 0:
-        self._tcs: None | System = None  # type: ignore[assignment]
         self.devices: list[Device] = []
-        self.device_by_id: dict[str, Device] = {}
-
-        self._setup_event_handlers()
+        self.device_by_id: dict[DeviceIdT, Device] = {}
 
-        load_schema(self, **self._schema)
+        self._zzz: MessageIndex | None = None  # MessageIndex()
 
     def __repr__(self) -> str:
-        return super().__str__()
-
-    def __str__(self) -> str:
-        return (self.hgi or HGI_DEV_ADDR).id
-
-    def _setup_event_handlers(self) -> None:  # HACK: for dev/test only
-        async def handle_sig_posix(sig):
-            """Handle signals on posix platform."""
-            _LOGGER.debug("Received a signal (%s), processing...", sig.name)
-
-            if sig == signal.SIGUSR1:
-                _LOGGER.info("Schema: \r\n%s", {self.tcs.id: self.tcs.schema})
-                _LOGGER.info("Params: \r\n%s", {self.tcs.id: self.tcs.params})
-                _LOGGER.info("Status: \r\n%s", {self.tcs.id: self.tcs.status})
-
-            elif sig == signal.SIGUSR2:
-                _LOGGER.info("Status: \r\n%s", {self.tcs.id: self.tcs.status})
+        if not self.ser_name:
+            return f"Gateway(input_file={self._input_file})"
+        return f"Gateway(port_name={self.ser_name}, port_config={self._port_config})"
 
-        super()._setup_event_handlers()
+    @property
+    def hgi(self) -> HgiGateway | None:
+        """Return the active HGI80-compatible gateway device, if known."""
+        if not self._transport:
+            return None
+        if device_id := self._transport.get_extra_info(SZ_ACTIVE_HGI):
+            return self.device_by_id.get(device_id)  # type: ignore[return-value]
+        return None
 
-        _LOGGER.debug("_setup_event_handlers(): Creating signal handlers...")
-        if os.name == "posix":  # full support
-            for sig in [signal.SIGUSR1, signal.SIGUSR2]:
-                self._loop.add_signal_handler(
-                    sig, lambda sig=sig: self._loop.create_task(handle_sig_posix(sig))
-                )
-        elif os.name == "nt":  # supported, but YMMV
-            _LOGGER.warning("Be aware, YMMV with Windows...")
-        else:  # unsupported
-            raise RuntimeError(f"Unsupported OS for this module: {os.name}")
+    async def start(
+        self,
+        /,
+        *,
+        start_discovery: bool = True,
+        cached_packets: dict[str, str] | None = None,
+    ) -> None:
+        """Start the Gateway and Initiate discovery as required."""
 
-    async def start(self, *, start_discovery: bool = True) -> None:
-        def initiate_discovery(dev_list, sys_list) -> None:
+        def initiate_discovery(dev_list: list[Device], sys_list: list[Evohome]) -> None:
             _LOGGER.debug("ENGINE: Initiating/enabling discovery...")
 
             # [d._start_discovery_poller() for d in devs]
@@ -418,84 +168,73 @@ class Gateway(Engine):
                 if system.dhw:
                     system.dhw._start_discovery_poller()
 
-        await super().start()
-
-        if not self.ser_name:  # wait until have processed the entire packet log...
-            await self.pkt_transport.get_extra_info(SZ_POLLER_TASK)
+        await set_pkt_logging_config(  # type: ignore[arg-type]
+            cc_console=self.config.reduce_processing >= DONT_CREATE_MESSAGES,
+            **self._packet_log,
+        )
 
-        elif start_discovery:  # source of packets is a serial port
-            initiate_discovery(self.devices, self.systems)
+        self.config.disable_discovery, disable_discovery = (
+            True,
+            self.config.disable_discovery,
+        )
 
-    async def stop(self) -> None:  # FIXME: a mess
-        """Cancel all outstanding tasks."""
-        # if self._engine_state is None:
-        #     self._pause()
+        load_schema(self, known_list=self._include, **self._schema)  # create faked too
 
-        # (t for t in self._tasks if not isinstance(t, asyncio.Task))
+        await super().start()  # TODO: do this *after* restore cache
+        if cached_packets:
+            await self._restore_cached_packets(cached_packets)
 
-        if [t.cancel() for t in self._tasks if not t.done()]:
-            await asyncio.gather(*(t for t in self._tasks if asyncio.isfuture(t)))
+        self.config.disable_discovery = disable_discovery
 
-        # this doesn't work...
-        for t in [t for t in self._tasks if isinstance(t, futures.Future)]:
-            try:
-                await asyncio.wrap_future(t, loop=self._loop)
-            except asyncio.CancelledError:
-                pass
+        if (
+            not self._disable_sending
+            and not self.config.disable_discovery
+            and start_discovery
+        ):
+            initiate_discovery(self.devices, self.systems)
 
-        # this doesn't work either
-        # for device in self.devices:
-        #     await device._stop_discovery_poller()
-        # for system in self.systems:
-        #     await system._stop_discovery_poller()
-        #     for zone in system.zones:
-        #         await zone._stop_discovery_poller()
+    async def stop(self) -> None:
+        """Stop the Gateway and tidy up."""
 
+        if self._zzz:
+            self._zzz.stop()
         await super().stop()
 
-    def _pause(self, *args, clear_state: bool = False) -> None:
-        """Pause the (unpaused) gateway."""
-
-        super()._pause(
-            self.config.disable_discovery, self.config.disable_sending, *args
-        )
-        self.config.disable_discovery = True
-        self.config.disable_sending = True
-
-        if clear_state:
-            self._clear_state()
+    def _pause(self, *args: Any) -> None:
+        """Pause the (unpaused) gateway (disables sending/discovery).
 
-    def _resume(self) -> tuple:
-        """Resume the (paused) gateway."""
+        There is the option to save other objects, as *args.
+        """
+        _LOGGER.debug("Gateway: Pausing engine...")
 
-        (
-            self.config.disable_discovery,
-            self.config.disable_sending,
-            *args,
-        ) = super()._resume()
+        self.config.disable_discovery, disc_flag = True, self.config.disable_discovery
 
-        return args  # type: ignore[return-value]
+        try:
+            super()._pause(disc_flag, *args)
+        except RuntimeError:
+            self.config.disable_discovery = disc_flag
+            raise
 
-    def _clear_state(self) -> None:
-        _LOGGER.warning("ENGINE: Clearing exisiting schema/state...")
+    def _resume(self) -> tuple[Any]:
+        """Resume the (paused) gateway (enables sending/discovery, if applicable).
 
-        self._tcs = None
-        self.devices = []
-        self.device_by_id = {}
+        Will restore other objects, as *args.
+        """
+        args: tuple[Any]
 
-    def get_state(self, include_expired: bool = False) -> tuple[dict, dict]:
+        _LOGGER.debug("Gateway: Resuming engine...")
 
-        (_LOGGER.warning if DEV_MODE else _LOGGER.debug)("ENGINE: Getting state...")
-        self._pause()
+        self.config.disable_discovery, *args = super()._resume()  # type: ignore[assignment]
 
-        result = self._get_state(include_expired=include_expired)
+        return args
 
-        self._resume()
-        (_LOGGER.warning if DEV_MODE else _LOGGER.debug)("ENGINE: Got schema/state.")
+    def get_state(
+        self, include_expired: bool = False
+    ) -> tuple[dict[str, Any], dict[str, str]]:
+        """Return the current schema & state (may include expired packets)."""
 
-        return result
+        self._pause()
 
-    def _get_state(self, include_expired: bool = False) -> tuple[dict, dict]:
         def wanted_msg(msg: Message, include_expired: bool = False) -> bool:
             if msg.code == Code._313F:
                 return msg.verb in (I_, RP)  # usu. expired, useful 4 back-back restarts
@@ -514,55 +253,102 @@ class Gateway(Engine):
         for system in self.systems:
             msgs.extend(list(system._msgs.values()))
             msgs.extend([m for z in system.zones for m in z._msgs.values()])
-            # msgs.extend([m for z in system.dhw for m in z._msgs.values()])
+            # msgs.extend([m for z in system.dhw for m in z._msgs.values()])  # TODO
+
+        if self._zzz:
+            pkts = {
+                f"{repr(msg._pkt)[:26]}": f"{repr(msg._pkt)[27:]}"
+                for msg in self._zzz.all(include_expired=True)
+                if wanted_msg(msg, include_expired=include_expired)
+            }
 
-        # BUG: assumes pkts have unique dtms: may not be true for contrived logs...
-        pkts = {
-            f"{repr(msg._pkt)[:26]}": f"{repr(msg._pkt)[27:]}"
-            for msg in msgs
-            if wanted_msg(msg, include_expired=include_expired)
-        }  # BUG: assumes pkts have unique dtms
+        else:
+            pkts = {  # BUG: assumes pkts have unique dtms: may be untrue for contrived logs
+                f"{repr(msg._pkt)[:26]}": f"{repr(msg._pkt)[27:]}"
+                for msg in msgs
+                if wanted_msg(msg, include_expired=include_expired)
+            }
+
+        self._resume()
 
         return self.schema, dict(sorted(pkts.items()))
 
-    async def set_state(self, packets, *, schema: None | dict = None) -> None:
-        # TODO: add a feature to exludede expired packets?
-        (_LOGGER.warning if DEV_MODE else _LOGGER.info)("ENGINE: Setting state...")
+    async def _restore_cached_packets(
+        self, packets: dict[str, str], _clear_state: bool = False
+    ) -> None:
+        """Restore cached packets (may include expired packets)."""
+
+        def clear_state() -> None:
+            _LOGGER.info("GATEWAY: Clearing existing schema/state...")
 
-        if schema is None:  # TODO: also for known_list (device traits)?
-            schema = shrink(self.schema)
+            # self._schema = {}
 
-        self._pause(clear_state=True)
+            self._tcs = None
+            self.devices = []
+            self.device_by_id = {}
 
-        load_schema(self, **schema)
-        await self._set_state(packets, schema=schema)
+            self._prev_msg = None
+            self._this_msg = None
 
-        self._resume()
-        (_LOGGER.warning if DEV_MODE else _LOGGER.info)("ENGINE: Set state.")
+        tmp_transport: RamsesTransportT  # mypy hint
+
+        _LOGGER.debug("GATEWAY: Restoring a cached packet log...")
+        self._pause()
+
+        if _clear_state:  # only intended for test suite use
+            clear_state()
+
+        # We do not always enforce the known_list whilst restoring a cache because
+        # if it does not contain a correctly configured HGI, a 'working' address is
+        # used (which could be different to the address in the cache) & wanted packets
+        # can be dropped unnecessarily.
+
+        enforce_include_list = bool(
+            self._enforce_known_list
+            and extract_known_hgi_id(
+                self._include, disable_warnings=True, strick_checking=True
+            )
+        )
 
-    async def _set_state(self, packets: dict, *, schema=None) -> None:
-        tmp_transport: asyncio.Transport
+        # The actual HGI address will be discovered when the actual transport was/is
+        # started up (usually before now)
 
-        pkt_receiver = (
-            self.msg_transport.get_extra_info(self.msg_transport.READER)
-            if self.msg_transport
-            else None
+        tmp_protocol = protocol_factory(
+            self._msg_handler,
+            disable_sending=True,
+            enforce_include_list=enforce_include_list,
+            exclude_list=self._exclude,
+            include_list=self._include,
         )
-        _, tmp_transport = self._create_pkt_stack(pkt_receiver, packet_dict=packets)
-        await tmp_transport.get_extra_info(SZ_POLLER_TASK)
 
-        # self.msg_transport._clear_write_buffer()  # TODO: shouldn't be needed
-        self.msg_protocol._prev_msg = None  # TODO: move to pause/resume?
+        tmp_transport = await transport_factory(
+            tmp_protocol,
+            packet_dict=packets,
+        )
+
+        await tmp_transport.get_extra_info(SZ_READER_TASK)
+
+        _LOGGER.debug("GATEWAY: Restored, resuming")
+        self._resume()
+
+    def _add_device(self, dev: Device) -> None:  # TODO: also: _add_system()
+        """Add a device to the gateway (called by devices during instantiation)."""
+
+        if dev.id in self.device_by_id:
+            raise LookupError(f"Device already exists: {dev.id}")
+
+        self.devices.append(dev)
+        self.device_by_id[dev.id] = dev
 
     def get_device(
         self,
-        dev_id: _DeviceIdT,
+        device_id: DeviceIdT,
         *,
-        msg: None | Message = None,
-        parent=None,
-        child_id=None,
-        is_sensor: None | bool = None,
-    ) -> Device:  # TODO: **schema) -> Device:  # may: LookupError
+        msg: Message | None = None,
+        parent: Parent | None = None,
+        child_id: str | None = None,
+        is_sensor: bool | None = None,
+    ) -> Device:  # TODO: **schema/traits) -> Device:  # may: LookupError
         """Return a device, create it if required.
 
         First, use the traits to create/update it, then pass it any msg to handle.
@@ -572,13 +358,13 @@ class Gateway(Engine):
         If a device is created, attach it to the gateway.
         """
 
-        def check_filter_lists(dev_id: _DeviceIdT) -> None:  # may: LookupError
+        def check_filter_lists(dev_id: DeviceIdT) -> None:  # may: LookupError
             """Raise an LookupError if a device_id is filtered out by a list."""
 
             if dev_id in self._unwanted:  # TODO: shouldn't invalidate a msg
                 raise LookupError(f"Can't create {dev_id}: it is unwanted or invalid")
 
-            if self.config.enforce_known_list and (
+            if self._enforce_known_list and (
                 dev_id not in self._include and dev_id != getattr(self.hgi, "id", None)
             ):
                 self._unwanted.append(dev_id)
@@ -594,14 +380,32 @@ class Gateway(Engine):
                     f" (if required, remove it from the {SZ_BLOCK_LIST})"
                 )
 
-        check_filter_lists(dev_id)
-        traits = SCH_TRAITS(self._include.get(dev_id, {}))
+        try:
+            check_filter_lists(device_id)
+        except LookupError:
+            # have to allow for GWY not being in known_list...
+            if device_id != self._protocol.hgi_id:
+                raise  # TODO: make parochial
+
+        dev = self.device_by_id.get(device_id)
 
-        dev = self.device_by_id.get(dev_id)
         if not dev:
-            dev = device_factory(self, Address(dev_id), msg=msg, **traits)
+            # voluptuous bug workaround: https://github.com/alecthomas/voluptuous/pull/524
+            _traits: dict[str, Any] = self._include.get(device_id, {})  # type: ignore[assignment]
+            _traits.pop("commands", None)
+
+            traits: dict[str, Any] = SCH_TRAITS(self._include.get(device_id, {}))
+
+            dev = device_factory(self, Address(device_id), msg=msg, **_traits)
+
+            if traits.get(SZ_FAKED):
+                if isinstance(dev, Fakeable):
+                    dev._make_fake()
+                else:
+                    _LOGGER.warning(f"The device is not fakeable: {dev}")
 
         # TODO: the exact order of the following may need refining...
+        # TODO: some will be done my devices themselves?
 
         # if schema:  # Step 2: Only controllers have a schema...
         #     dev._update_schema(**schema)  # TODO: schema/traits
@@ -609,19 +413,34 @@ class Gateway(Engine):
         if parent or child_id:
             dev.set_parent(parent, child_id=child_id, is_sensor=is_sensor)
 
-        if traits.get(SZ_FAKED):
-            if isinstance(dev, Fakeable):
-                dev._make_fake()
-            else:
-                _LOGGER.warning(f"The device is not fakable: {dev}")
-
-        if msg:
-            dev._handle_msg(msg)
+        # if msg:
+        #     dev._handle_msg(msg)
 
         return dev
 
+    def fake_device(
+        self,
+        device_id: DeviceIdT,
+        create_device: bool = False,
+    ) -> Device:
+        """Create a faked device."""
+
+        if not is_valid_dev_id(device_id):
+            raise TypeError(f"The device id is not valid: {device_id}")
+
+        if not create_device and device_id not in self.device_by_id:
+            raise LookupError(f"The device id does not exist: {device_id}")
+        elif create_device and device_id not in self.known_list:
+            raise LookupError(f"The device id is not in the known_list: {device_id}")
+
+        if (dev := self.get_device(device_id)) and isinstance(dev, Fakeable):
+            dev._make_fake()
+            return dev
+
+        raise TypeError(f"The device is not fakable: {device_id}")
+
     @property
-    def tcs(self) -> Optional[System]:
+    def tcs(self) -> Evohome | None:
         """Return the primary TCS, if any."""
 
         if self._tcs is None and self.systems:
@@ -629,7 +448,7 @@ class Gateway(Engine):
         return self._tcs
 
     @property
-    def known_list(self) -> dict:
+    def known_list(self) -> DeviceListT:
         """Return the working known_list (a superset of the provided known_list).
 
         Unlike orphans, which are always instantiated when a schema is loaded, these
@@ -640,15 +459,15 @@ class Gateway(Engine):
         result = self._include  # could be devices here, not (yet) in gwy.devices
         result.update(
             {
-                d.id: {k: d.traits[k] for k in (SZ_CLASS, SZ_ALIAS, SZ_FAKED)}
+                d.id: {k: d.traits[k] for k in (SZ_CLASS, SZ_ALIAS, SZ_FAKED)}  # type: ignore[misc]
                 for d in self.devices
-                if not self.config.enforce_known_list or d.id in self._include
+                if not self._enforce_known_list or d.id in self._include
             }
         )
         return result
 
     @property
-    def system_by_id(self) -> dict:
+    def system_by_id(self) -> dict[DeviceIdT, Evohome]:
         return {
             d.id: d.tcs
             for d in self.devices
@@ -656,11 +475,11 @@ class Gateway(Engine):
         }  # why something so simple look so messy
 
     @property
-    def systems(self) -> list:
+    def systems(self) -> list[Evohome]:
         return list(self.system_by_id.values())
 
     @property
-    def _config(self) -> dict:
+    def _config(self) -> dict[str, Any]:
         """Return the working configuration.
 
         Includes:
@@ -673,28 +492,30 @@ class Gateway(Engine):
         return {
             "_gateway_id": self.hgi.id if self.hgi else None,
             SZ_MAIN_TCS: self.tcs.id if self.tcs else None,
-            SZ_CONFIG: {SZ_ENFORCE_KNOWN_LIST: self.config.enforce_known_list},
+            SZ_CONFIG: {SZ_ENFORCE_KNOWN_LIST: self._enforce_known_list},
             SZ_KNOWN_LIST: self.known_list,
             SZ_BLOCK_LIST: [{k: v} for k, v in self._exclude.items()],
-            "_unwanted": sorted(self.pkt_protocol._unwanted),
-            "_unwanted_alt": sorted(self._unwanted),
+            "_unwanted": sorted(self._unwanted),
         }
 
     @property
-    def schema(self) -> dict:
+    def schema(self) -> dict[str, Any]:
         """Return the global schema.
 
+        This 'active' schema may exclude non-present devices from the configured schema
+        that was loaded during initialisation.
+
         Orphans are devices that 'exist' but don't yet have a place in the schema
-        hierachy (if ever): therefore, they are instantiated when the schema is loaded,
+        hierarchy (if ever): therefore, they are instantiated when the schema is loaded,
         just like the other devices in the schema.
         """
 
-        schema = {SZ_MAIN_TCS: self.tcs.ctl.id if self.tcs else None}
+        schema: dict[str, Any] = {SZ_MAIN_TCS: self.tcs.ctl.id if self.tcs else None}
 
         for tcs in self.systems:
             schema[tcs.ctl.id] = tcs.schema
 
-        schema[f"{SZ_ORPHANS}_heat"] = sorted(
+        dev_list: list[DeviceIdT] = sorted(
             [
                 d.id
                 for d in self.devices
@@ -703,67 +524,110 @@ class Gateway(Engine):
                 and d._is_present
             ]
         )
+        schema[f"{SZ_ORPHANS}_heat"] = dev_list
 
-        schema[f"{SZ_ORPHANS}_hvac"] = sorted(
+        dev_list = sorted(
             [d.id for d in self.devices if isinstance(d, DeviceHvac) and d._is_present]
         )
+        schema[f"{SZ_ORPHANS}_hvac"] = dev_list
 
         return schema
 
     @property
-    def params(self) -> dict:
+    def params(self) -> dict[str, Any]:
         return {SZ_DEVICES: {d.id: d.params for d in sorted(self.devices)}}
 
     @property
-    def status(self) -> dict:
-        return {SZ_DEVICES: {d.id: d.status for d in sorted(self.devices)}}
+    def status(self) -> dict[str, Any]:
+        tx_rate = self._transport.get_extra_info("tx_rate") if self._transport else None
+        return {
+            SZ_DEVICES: {d.id: d.status for d in sorted(self.devices)},
+            "_tx_rate": tx_rate,
+        }
 
-    def send_cmd(  # FIXME
-        self, cmd: Command, callback: Callable = None, **kwargs
-    ) -> futures.Future:
-        """Send a command with the option to return any response via callback."""
+    def _msg_handler(self, msg: Message) -> None:
+        """A callback to handle messages from the protocol stack."""
+        # TODO: Remove this
+        # # HACK: if CLI, double-logging with client.py proc_msg() & setLevel(DEBUG)
+        # if (log_level := _LOGGER.getEffectiveLevel()) < logging.INFO:
+        #     _LOGGER.info(msg)
+        # elif log_level <= logging.INFO and not (
+        #     msg.verb == RQ and msg.src.type == DEV_TYPE_MAP.HGI
+        # ):
+        #     _LOGGER.info(msg)
 
-        if self.config.disable_sending:
-            raise RuntimeError("sending is disabled")
+        super()._msg_handler(msg)
 
-        fut = super().send_cmd(cmd, callback, **kwargs)
+        # TODO: ideally remove this feature...
+        assert self._this_msg  # mypy check
 
-        self._tasks = [t for t in self._tasks if not t.done()]
-        self._tasks.append(fut)
-        return fut
+        if self._prev_msg and detect_array_fragment(self._this_msg, self._prev_msg):
+            msg._pkt._force_has_array()  # may be an array of length 1
+            msg._payload = self._prev_msg.payload + (
+                msg.payload if isinstance(msg.payload, list) else [msg.payload]
+            )
 
-    def fake_device(
-        self,
-        device_id: _DeviceIdT,
-        create_device: bool = False,
-        start_binding: bool = False,
-    ) -> Device:
-        """Create a faked device, and optionally set it to binding mode.
+        process_msg(self, msg)
 
-        Will make any neccesary changed to the device lists.
+    def send_cmd(
+        self,
+        cmd: Command,
+        /,
+        *,
+        gap_duration: float = DEFAULT_GAP_DURATION,
+        num_repeats: int = DEFAULT_NUM_REPEATS,
+        priority: Priority = Priority.DEFAULT,
+        timeout: float = DEFAULT_SEND_TIMEOUT,
+        wait_for_reply: bool | None = DEFAULT_WAIT_FOR_REPLY,
+    ) -> asyncio.Task[Packet]:
+        """Wrapper to schedule an async_send_cmd() and return the Task.
+
+        num_repeats:  0 = send once, 1 = send twice, etc.
+        gap_duration: the gap between repeats (in seconds)
+        priority:     the priority of the command
         """
-        # TODO: what about using the HGI
 
-        if not is_valid_dev_id(device_id):
-            raise TypeError(f"The device id is not valid: {device_id}")
-
-        if create_device and device_id in self.device_by_id:
-            raise LookupError(f"The device id already exists: {device_id}")
-        elif not create_device and device_id not in self.device_by_id:
-            raise LookupError(f"The device id does not exist: {device_id}")
+        coro = self.async_send_cmd(
+            cmd,
+            gap_duration=gap_duration,
+            num_repeats=num_repeats,
+            priority=priority,
+            timeout=timeout,
+            wait_for_reply=wait_for_reply,
+        )
 
-        if self.config.enforce_known_list and device_id not in self._include:
-            self._include[device_id] = {}
-        elif device_id in self._exclude:
-            del self._exclude[device_id]
+        task = self._loop.create_task(coro)
+        self.add_task(task)
+        return task
 
-        if (dev := self.get_device(device_id)) and isinstance(dev, Fakeable):
-            return dev._make_fake(bind=start_binding)
-        raise TypeError(f"The device is not fakable: {device_id}")
+    async def async_send_cmd(
+        self,
+        cmd: Command,
+        /,
+        *,
+        gap_duration: float = DEFAULT_GAP_DURATION,
+        num_repeats: int = DEFAULT_NUM_REPEATS,
+        priority: Priority = Priority.DEFAULT,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+        timeout: float = DEFAULT_SEND_TIMEOUT,
+        wait_for_reply: bool | None = DEFAULT_WAIT_FOR_REPLY,
+    ) -> Packet:
+        """Send a Command and return the corresponding (echo or reply) Packet.
+
+        If wait_for_reply is True (*and* the Command has a rx_header), return the
+        reply Packet. Otherwise, simply return the echo Packet.
+
+        If the expected Packet can't be returned, raise:
+            ProtocolSendFailed: tried to Tx Command, but didn't get echo/reply
+            ProtocolError:      didn't attempt to Tx Command for some reason
+        """
 
-    def add_task(self, fnc, *args, delay=None, period=None, **kwargs) -> asyncio.Task:
-        """Start a task after delay seconds and then repeat it every period seconds."""
-        self._tasks = [t for t in self._tasks if not t.done()]
-        task = schedule_task(fnc, *args, delay=delay, period=period, **kwargs)
-        self._tasks.append(task)
-        return task
+        return await super().async_send_cmd(
+            cmd,
+            gap_duration=gap_duration,
+            num_repeats=num_repeats,
+            priority=priority,
+            max_retries=max_retries,
+            timeout=timeout,
+            wait_for_reply=wait_for_reply,
+        )  # may: raise ProtocolError/ProtocolSendFailed