motorcortex-python 1.0.0rc1__py3-none-any.whl

motorcortex/_request_utils.py

@@ -0,0 +1,314 @@
+#
+#   Developer: Alexey Zakharov (alexey.zakharov@vectioneer.com)
+#   All rights reserved. Copyright (c) 2026 VECTIONEER.
+#
+
+"""Pure helpers used by :class:`motorcortex.request.Request`.
+
+None of these touch the socket — they're either kwarg massaging, thread-
+sync waits, or filesystem cache I/O. Extracted so they can be unit-tested
+without a live server. ``Request`` keeps thin static-method shims that
+delegate here, so existing callers and tests stay on the public API.
+"""
+
+from __future__ import annotations
+
+import base64
+import glob
+import hashlib
+import json
+import os
+import tempfile
+from threading import Event
+from typing import Any, Callable, Optional, Tuple
+
+from motorcortex.exceptions import McxConnectionError
+from motorcortex.setup_logger import logger
+
+
+_CACHE_PREFIX = "mcx-python-pt"
+
+
+def parse_connect_kwargs(
+        conn_timeout_ms: int = 0,
+        timeout_ms: Optional[int] = None,
+        recv_timeout_ms: Optional[int] = None,
+        certificate: Optional[str] = None,
+        login: Optional[str] = None,
+        password: Optional[str] = None,
+        state_update: Optional[Callable] = None,
+        **kwargs: Any,
+) -> Tuple[int, Optional[int], Optional[str], Optional[Callable]]:
+    """Normalise the kwargs bag ``Request.connect()`` accepts.
+
+    The only non-trivial rule is that ``timeout_ms`` promotes to
+    ``conn_timeout_ms`` when the latter wasn't passed explicitly. Any
+    extra kwargs (``login``, ``password``, ``unknown_knob``, ...) are
+    swallowed so the caller can pass a superset of keys without
+    triggering a ``TypeError``.
+    """
+    if timeout_ms and not conn_timeout_ms:
+        conn_timeout_ms = timeout_ms
+    return conn_timeout_ms, recv_timeout_ms, certificate, state_update
+
+
+def wait_for_connection(
+        event: Event,
+        timeout_sec: float,
+        is_connected_fn: Optional[Callable[[], bool]] = None,
+) -> bool:
+    """Block on ``event`` and report whether a connection actually succeeded.
+
+    Args:
+        event: ``Event`` that connect-side code sets once the handshake
+            either completed or failed.
+        timeout_sec: Max wait. ``<= 0`` means wait forever (passed through
+            to ``event.wait()`` with no timeout).
+        is_connected_fn: Optional status probe. The event fires on *any*
+            terminal transition (success, failure, close), so the probe
+            disambiguates — when provided, must return ``True`` if the
+            client is genuinely connected.
+
+    Returns:
+        ``True`` on success.
+
+    Raises:
+        McxConnectionError: ``event`` didn't fire within
+            ``timeout_sec`` (dial timeout), OR it fired but
+            ``is_connected_fn()`` returned falsy (peer dropped /
+            handshake failed). Both are connect-side transport
+            failures per the documented contract in
+            ARCHITECTURE.md §2b, so they share one exception type.
+    """
+    logger.debug("[REQUEST-WAIT] waitForConnection started with timeout=%ss", timeout_sec)
+
+    if timeout_sec <= 0:
+        logger.debug("[REQUEST-WAIT] Waiting indefinitely for connection event")
+        result = event.wait()
+    else:
+        logger.debug("[REQUEST-WAIT] Waiting up to %ss for connection event", timeout_sec)
+        result = event.wait(timeout_sec)
+
+    if not result:
+        logger.error("[REQUEST-WAIT] Connection timeout after %ss - no event received", timeout_sec)
+        raise McxConnectionError(f"Connection timeout after {timeout_sec}s")
+
+    logger.debug("[REQUEST-WAIT] Event received - checking connection status")
+
+    if is_connected_fn is not None:
+        connected = is_connected_fn()
+        logger.debug("[REQUEST-WAIT] Connection status check: connected=%s", connected)
+        if not connected:
+            logger.error("[REQUEST-WAIT] Event was set but connection failed or was closed")
+            raise McxConnectionError("Connection failed or was closed before completion")
+    else:
+        logger.debug("[REQUEST-WAIT] No connection status check function provided")
+
+    logger.debug("[REQUEST-WAIT] Connection successfully established")
+    return True
+
+
+def fetch_parameter_tree(
+        tree_hash: int,
+        protobuf_types: Any,
+        socket: Any,
+        url: Optional[str] = None,
+) -> Any:
+    """Fetch the parameter tree, hitting the local cache first.
+
+    Used by ``Request.getParameterTree`` as the executor-pool callable.
+    Kept here (not on ``Request``) so the orchestration between
+    :func:`parameter_tree_cache_path`, :func:`load_parameter_tree_file`,
+    :func:`send_and_recv`, and :func:`save_parameter_tree_file` is unit
+    testable with stubs.
+
+    Args:
+        tree_hash: Integer from ``getParameterTreeHash`` — part of the
+            cache filename.
+        protobuf_types: ``MessageTypes`` used to build the request
+            message and decode the reply.
+        socket: pynng ``Req0``-ish handle passed to
+            :func:`send_and_recv`.
+        url: Connection URL — hashed into the cache filename so two
+            engines with the same ``tree_hash`` don't collide.
+
+    Returns:
+        A ``ParameterTreeMsg`` — either the cached one or the freshly
+        fetched one (also persisted to the cache).
+    """
+    path = parameter_tree_cache_path(url, tree_hash)
+    cached = load_parameter_tree_file(path, protobuf_types)
+    if cached:
+        logger.debug("[REQUEST] Found parameter tree in the cache")
+        return cached
+    logger.debug("[REQUEST] Failed to find parameter tree in the cache")
+
+    request_msg = protobuf_types.createType("motorcortex.GetParameterTreeMsg")
+    handle = send_and_recv(socket, protobuf_types.encode(request_msg), protobuf_types)
+    return save_parameter_tree_file(path, handle)
+
+
+def send_and_recv(
+        socket: Any,
+        encoded_msg: Any,
+        protobuf_types: Optional[Any],
+) -> Any:
+    """One-shot synchronous request/reply over an nng Req0 socket.
+
+    Opens a new nng context on ``socket``, sends ``encoded_msg``, waits
+    for a reply, optionally decodes it through ``protobuf_types.decode``,
+    and closes the context. Errors from nng propagate — they're logged
+    for diagnostics and re-raised so the caller's ``Future`` sees the
+    failure instead of swallowing it.
+
+    Args:
+        socket: A pynng ``Req0``-ish object exposing ``new_context()``.
+            The context it returns must support ``send()``, ``recv()``,
+            ``close()``.
+        encoded_msg: Bytes to send.
+        protobuf_types: If truthy, decoded wire reply is returned.
+            Otherwise the raw buffer is returned.
+
+    Returns:
+        Decoded reply (when ``protobuf_types`` supplied and buffer is
+        non-empty), raw buffer, or ``None`` if nng returned empty.
+    """
+    ctx = socket.new_context()
+    try:
+        ctx.send(encoded_msg)
+        buffer = ctx.recv()
+        if buffer:
+            if protobuf_types:
+                return protobuf_types.decode(buffer)
+            return buffer
+    except Exception as e:
+        logger.error("[SEND] Error during send/recv: %s: %s", type(e).__name__, e)
+        raise
+    finally:
+        ctx.close()
+    return None
+
+
+def parameter_tree_cache_path(url: Optional[str], tree_hash: int) -> str:
+    """Build the temp-dir path that caches the parameter tree for a given
+    ``(url, tree_hash)`` pair.
+
+    Before 0.25.8 the cache was keyed only on ``tree_hash``, so two
+    different engines that happened to produce the same hash value would
+    read each other's cached tree. This function now hashes the URL into
+    the filename so engines with different URLs get distinct cache files.
+    A short md5 prefix keeps the path manageable.
+
+    ``url`` may be ``None`` (e.g. before the Request has finished
+    connecting) — in that case we fall back to a ``nourl`` placeholder,
+    which still preserves the hash-based deduplication for a single
+    process.
+    """
+    url_key = hashlib.md5((url or "nourl").encode("utf-8")).hexdigest()[:8]
+    return os.path.join(
+        tempfile.gettempdir(), f"{_CACHE_PREFIX}-{url_key}-{tree_hash}"
+    )
+
+
+def _purge_stale_cache_files(path: str) -> None:
+    """Remove previous cache files for the same URL.
+
+    Called by :func:`save_parameter_tree_file` after a successful write.
+    The filename layout is ``mcx-python-pt-<url-md5>-<tree-hash>``; this
+    helper globs the ``mcx-python-pt-<url-md5>-`` prefix of the just-
+    written file and unlinks any sibling that doesn't match the current
+    ``tree-hash``. That keeps the cache directory at one file per URL
+    in steady state instead of accumulating forever as servers cycle
+    through tree-hash values.
+
+    Temp files from concurrent :func:`tempfile.mkstemp` writers are
+    skipped — their names contain a ``.`` right after the hash, which
+    canonical cache filenames never do.
+    """
+    directory, basename = os.path.split(path)
+    last_dash = basename.rfind("-")
+    if last_dash == -1:
+        return
+    url_prefix = basename[:last_dash + 1]  # "mcx-python-pt-<url-md5>-"
+    pattern = os.path.join(directory or ".", url_prefix + "*")
+    for match in glob.glob(pattern):
+        entry = os.path.basename(match)
+        if entry == basename:
+            continue
+        # mkstemp injects ".<random>.tmp" after the hash — skip so we
+        # don't race a concurrent writer to its own temp file.
+        if "." in entry[len(url_prefix):]:
+            continue
+        try:
+            os.unlink(match)
+            logger.debug("[REQUEST] Evicted stale cache file: %s", entry)
+        except OSError:
+            pass
+
+
+def save_parameter_tree_file(path: str, parameter_tree: Any) -> Any:
+    """Serialise ``parameter_tree`` to a local JSON cache file.
+
+    Writes a ``{"md5": ..., "data": base64}`` envelope at ``path``. The
+    md5 is over the **base64** payload (not the raw protobuf bytes), so
+    tampering with ``data`` alone invalidates the cache on next load.
+
+    The write is atomic: the envelope goes to a sibling temp file in the
+    same directory, then ``os.replace`` moves it into place. On POSIX
+    this is a single rename syscall, so concurrent readers either see
+    the old complete file or the new complete file — never a half-written
+    mix that would fail md5 on the next ``load_parameter_tree_file``.
+    """
+    logger.debug("[REQUEST] Saved parameter tree to the cache")
+    base64_data = base64.b64encode(parameter_tree.SerializeToString())
+    envelope = {
+        "md5": hashlib.md5(base64_data).hexdigest(),
+        "data": base64_data.decode("utf-8"),
+    }
+
+    directory = os.path.dirname(path) or "."
+    fd, tmp_path = tempfile.mkstemp(
+        prefix=os.path.basename(path) + ".",
+        suffix=".tmp",
+        dir=directory,
+    )
+    try:
+        with os.fdopen(fd, "w") as outfile:
+            outfile.write(json.dumps(envelope))
+        os.replace(tmp_path, path)
+    except Exception:
+        # Clean up the temp file if the rename never ran.
+        if os.path.exists(tmp_path):
+            try:
+                os.unlink(tmp_path)
+            except OSError:
+                pass
+        raise
+    # Cache is now durable; best-effort sweep of stale same-URL files.
+    _purge_stale_cache_files(path)
+    return parameter_tree
+
+
+def load_parameter_tree_file(path: str, protobuf_types: Any) -> Optional[Any]:
+    """Inverse of :func:`save_parameter_tree_file` with md5 verification.
+
+    Returns ``None`` when the file is missing, malformed, or fails the
+    md5 check. Malformed here means: not valid JSON, missing keys, or
+    md5 mismatch. Any other exception propagates (disk errors, etc.).
+    """
+    logger.debug("[REQUEST] Loaded parameter tree from the cache")
+    if not os.path.exists(path):
+        return None
+
+    with open(path, "r") as outfile:
+        json_data = json.load(outfile)
+
+    if not json_data or "md5" not in json_data or "data" not in json_data:
+        return None
+
+    if hashlib.md5(json_data["data"].encode()).hexdigest() != json_data["md5"]:
+        return None
+
+    msg = protobuf_types.createType("motorcortex.ParameterTreeMsg")
+    msg.ParseFromString(base64.b64decode(json_data["data"]))
+    return msg

motorcortex/_subscribe_dispatch.py

@@ -0,0 +1,90 @@
+#
+#   Developer: Alexey Zakharov (alexey.zakharov@vectioneer.com)
+#   All rights reserved. Copyright (c) 2026 VECTIONEER.
+#
+
+"""Pure frame-parsing and dispatch helpers for :mod:`motorcortex.subscribe`.
+
+These used to be inlined in ``Subscribe.__run``. Moving them out lets us
+unit-test the wire format parser and the protocol-version dispatch
+without standing up a real subscribe socket — in particular the
+"unknown protocol version" branch that real servers never produce.
+
+Wire frame layout:
+    bytes [0..2]  little-endian 24-bit subscription id
+    byte   3      protocol version (0 = MULTI_TIMESTAMP, 1 = SINGLE_TIMESTAMP)
+    bytes [4..]   payload passed to ``Subscription._updateProtocol{0,1}``
+"""
+
+from __future__ import annotations
+
+from typing import Any, Mapping, Tuple
+
+from motorcortex.setup_logger import logger
+
+
+_HEADER_SIZE = 4
+
+
+def parse_frame_header(buffer: bytes) -> Tuple[int, int]:
+    """Decode the leading 4-byte frame header.
+
+    Args:
+        buffer: Raw bytes received on the subscribe socket. Must be at
+            least ``_HEADER_SIZE`` long; shorter inputs raise IndexError.
+
+    Returns:
+        ``(sub_id, protocol_version)``.
+    """
+    sub_id = buffer[0] + (buffer[1] << 8) + (buffer[2] << 16)
+    protocol_version = buffer[3]
+    return sub_id, protocol_version
+
+
+def dispatch_frame(buffer: bytes, subscriptions: Mapping[int, Any]) -> None:
+    """Route a received wire frame to the matching ``Subscription``.
+
+    Drops frames that fail any of:
+    - empty / shorter than the 4-byte header,
+    - sub_id not present in ``subscriptions``,
+    - protocol version ≠ 0 and ≠ 1.
+
+    Dropped frames are logged at ``error`` (unknown protocol / short frame)
+    or ``debug`` (unknown sub_id — can happen briefly after unsubscribe).
+    """
+    if not buffer:
+        return
+    if len(buffer) < _HEADER_SIZE:
+        logger.error(
+            '[SUBSCRIBE-DISPATCH] Frame shorter than header (%d bytes)',
+            len(buffer),
+        )
+        return
+
+    sub_id, protocol_version = parse_frame_header(buffer)
+    sub = subscriptions.get(sub_id)
+
+    if sub is None:
+        logger.debug(
+            '[SUBSCRIBE-DISPATCH] Received data for unknown subscription id: %s',
+            sub_id,
+        )
+        return
+
+    payload = buffer[_HEADER_SIZE:]
+    payload_length = len(buffer) - _HEADER_SIZE
+
+    if protocol_version == 1:
+        sub._updateProtocol1(payload, payload_length)
+    elif protocol_version == 0:
+        sub._updateProtocol0(payload, payload_length)
+    else:
+        logger.error(
+            '[SUBSCRIBE-DISPATCH] Unknown protocol version: %s for sub_id: %s',
+            protocol_version, sub_id,
+        )
+        # Bump the subscription's dropped-frame count so callers can
+        # observe the silent discard via ``Subscription.droppedFrameCount()``.
+        recorder = getattr(sub, '_recordDroppedFrame', None)
+        if recorder is not None:
+            recorder()

motorcortex/exceptions.py

@@ -0,0 +1,65 @@
+#
+#   Developer: Alexey Zakharov (alexey.zakharov@vectioneer.com)
+#   All rights reserved. Copyright (c) 2026 VECTIONEER.
+#
+
+"""Typed exceptions raised by the motorcortex library.
+
+The library follows a deliberate split between **transport** errors
+(which raise) and **protocol-level** status (which lives on the reply
+message and is checked by the caller):
+
+- **Transport errors raise** an :class:`McxError` subclass. Use when
+  the socket is closed, the server never answers, or the dial itself
+  fails. Caller can't do anything with the reply — there isn't one.
+- **Protocol-level status stays on the reply.** ``reply.get().status``
+  carries codes like ``WRONG_PASSWORD``, ``WRONG_PARAMETER_PATH``,
+  ``READ_ONLY_MODE``. Caller inspects if they care; nothing raises.
+
+Every exception here inherits from ``RuntimeError`` so existing
+``try: ... except RuntimeError: ...`` code keeps catching
+motorcortex failures without modification.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+
+class McxError(RuntimeError):
+    """Base class for all motorcortex library errors.
+
+    Inherits from ``RuntimeError`` so code that was written against the
+    pre-typed-exception era (``except RuntimeError``) keeps working
+    when the library swaps in more specific types.
+    """
+
+
+class McxConnectionError(McxError):
+    """Transport failure: socket is closed, server is unreachable,
+    dial was rejected, or a request was made on a connection that
+    never came up.
+    """
+
+
+class McxLoginError(McxError):
+    """Login RPC returned a non-OK status during
+    :func:`motorcortex.connect` or
+    :meth:`motorcortex.Session.connect`.
+
+    The engine's numeric status code is available as :attr:`status`
+    so callers can distinguish ``WRONG_PASSWORD`` from
+    ``READ_ONLY_MODE`` without parsing the message text.
+    """
+
+    def __init__(self, message: str, status: Optional[int] = None) -> None:
+        super().__init__(message)
+        self.status: Optional[int] = status
+
+
+class McxTimeout(McxError, TimeoutError):
+    """A reply didn't arrive within the requested timeout.
+
+    Dual-bases ``McxError`` and the stdlib ``TimeoutError`` so either
+    ``except McxError`` or ``except TimeoutError`` catches it.
+    """

motorcortex/init_threads.py

@@ -0,0 +1,103 @@
+#!/usr/bin/python3
+
+#
+#   Developer: Alexey Zakharov (alexey.zakharov@vectioneer.com)
+#   All rights reserved. Copyright (c) 2016-2026 VECTIONEER.
+#
+
+"""NNG thread-count initialization.
+
+``init_nng_threads`` is called automatically once from
+``motorcortex/__init__.py`` — it **must** run before any pynng socket
+is created, because NNG latches these values on its first internal
+init and ignores subsequent ``nng_init_set_parameter`` calls. The
+import-time auto-call is what guarantees that ordering for users who
+only touch NNG through motorcortex.
+
+Values are process-wide, so anything else using pynng in the same
+process inherits them. The hardcoded defaults (task=2, expire=1,
+poller=1, resolver=1) are tuned for the request/subscribe workload;
+most callers never need to tune them.
+
+Ops-side tuning is via env vars, read at import time — no code
+change needed:
+
+    MCX_NNG_TASK_THREADS       default 2
+    MCX_NNG_EXPIRE_THREADS     default 1
+    MCX_NNG_POLLER_THREADS     default 1
+    MCX_NNG_RESOLVER_THREADS   default 1
+
+Any explicit keyword argument to ``init_nng_threads(...)`` takes
+precedence over the env value. Env bad-values (non-int, negative)
+fall back to the default with a warning.
+"""
+
+import os
+
+from pynng._nng import lib  # type: ignore[import-untyped]
+from motorcortex.setup_logger import logger
+
+
+_DEFAULTS = {
+    "task": 2,
+    "expire": 1,
+    "poller": 1,
+    "resolver": 1,
+}
+
+_ENV_NAMES = {
+    "task": "MCX_NNG_TASK_THREADS",
+    "expire": "MCX_NNG_EXPIRE_THREADS",
+    "poller": "MCX_NNG_POLLER_THREADS",
+    "resolver": "MCX_NNG_RESOLVER_THREADS",
+}
+
+
+def _resolve(name: str, override: int | None) -> int:
+    """Pick the value for one knob: explicit arg > env var > default."""
+    if override is not None:
+        return override
+    raw = os.environ.get(_ENV_NAMES[name])
+    if raw is None:
+        return _DEFAULTS[name]
+    try:
+        value = int(raw)
+        if value < 1:
+            raise ValueError("must be >= 1")
+        return value
+    except ValueError as e:
+        logger.warning(
+            "[INIT-THREADS] %s=%r invalid (%s); using default %d",
+            _ENV_NAMES[name], raw, e, _DEFAULTS[name],
+        )
+        return _DEFAULTS[name]
+
+
+def init_nng_threads(
+    task: int | None = None,
+    expire: int | None = None,
+    poller: int | None = None,
+    resolver: int | None = None,
+) -> None:
+    """Set NNG thread-pool sizes. Must run before any pynng socket.
+
+    Any argument left as ``None`` is resolved from the matching
+    ``MCX_NNG_*_THREADS`` env var, falling back to the hardcoded
+    default.
+    """
+    task_n = _resolve("task", task)
+    expire_n = _resolve("expire", expire)
+    poller_n = _resolve("poller", poller)
+    resolver_n = _resolve("resolver", resolver)
+
+    try:
+        lib.nng_init_set_parameter(lib.NNG_INIT_NUM_TASK_THREADS, task_n)
+        lib.nng_init_set_parameter(lib.NNG_INIT_NUM_EXPIRE_THREADS, expire_n)
+        lib.nng_init_set_parameter(lib.NNG_INIT_NUM_POLLER_THREADS, poller_n)
+        lib.nng_init_set_parameter(lib.NNG_INIT_NUM_RESOLVER_THREADS, resolver_n)
+        logger.debug(
+            "[INIT-THREADS] task=%d expire=%d poller=%d resolver=%d",
+            task_n, expire_n, poller_n, resolver_n,
+        )
+    except AttributeError:
+        logger.error("[INIT-THREADS] Cannot adjust thread count: interface unavailable")